2026-02-22T21:38:58.740 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-22T21:38:58.758 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T21:38:58.836 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096 branch: wip-yuri6-testing-2026-02-22-1629-squid description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-workunit-policy-none} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '64096' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode name: yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-yuri6-testing-2026-02-22-1629-squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 rbd mirror image policy type: none client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 selinux: allowlist: - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri6-testing-2026-02-22-1629-squid sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9196 sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri6-testing-2026-02-22-1629-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 targets: trial048.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJJMis5tQRplKOa2XUKywCRJIwG5KxOlyoxvtJs22BJAkAwOpZt/4EmbZE9DgzvNfdZLt0tfEFyJGaAph1qzPS0= trial104.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMrJCFPlCkMRhmAELVBPerDyM3cOH9yMWaJ5BdKzAXf9e5uCF8/sM2a8qcWXicwuezEoql2WVcFQ7Alnj+tRo3c= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror.sh env: CEPH_ARGS: '' RBD_MIRROR_INSTANCES: '4' RBD_MIRROR_MODE: journal RBD_MIRROR_USE_EXISTING_CLUSTER: '1' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-22_19:35:38 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653 2026-02-22T21:38:58.836 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa; will attempt to use it 2026-02-22T21:38:58.837 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa/tasks 2026-02-22T21:38:58.837 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-22T21:38:58.838 INFO:teuthology.task.internal:Checking packages... 2026-02-22T21:38:58.838 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f90f4eaebb28b619cf76e80cbbe997656da69579' 2026-02-22T21:38:58.838 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-22T21:38:58.838 INFO:teuthology.packaging:ref: None 2026-02-22T21:38:58.839 INFO:teuthology.packaging:tag: None 2026-02-22T21:38:58.839 INFO:teuthology.packaging:branch: wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T21:38:58.839 INFO:teuthology.packaging:sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:38:58.839 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T21:38:58.955 INFO:teuthology.task.internal:Found packages for ceph version 19.2.3-689.gf90f4eae 2026-02-22T21:38:58.956 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-22T21:38:58.970 INFO:teuthology.task.internal:no buildpackages task found 2026-02-22T21:38:58.970 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-22T21:38:58.983 INFO:teuthology.task.internal:Saving configuration 2026-02-22T21:38:58.998 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-22T21:38:59.010 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-22T21:38:59.083 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial048.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 21:35:24.345402', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJJMis5tQRplKOa2XUKywCRJIwG5KxOlyoxvtJs22BJAkAwOpZt/4EmbZE9DgzvNfdZLt0tfEFyJGaAph1qzPS0='} 2026-02-22T21:38:59.140 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial104.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 21:35:24.346271', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMrJCFPlCkMRhmAELVBPerDyM3cOH9yMWaJ5BdKzAXf9e5uCF8/sM2a8qcWXicwuezEoql2WVcFQ7Alnj+tRo3c='} 2026-02-22T21:38:59.140 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-22T21:38:59.155 INFO:teuthology.task.internal:roles: ubuntu@trial048.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-22T21:38:59.155 INFO:teuthology.task.internal:roles: ubuntu@trial104.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-22T21:38:59.155 INFO:teuthology.run_tasks:Running task console_log... 2026-02-22T21:38:59.292 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x733f8254c040>, signals=[15]) 2026-02-22T21:38:59.292 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-22T21:38:59.305 INFO:teuthology.task.internal:Opening connections... 2026-02-22T21:38:59.305 DEBUG:teuthology.task.internal:connecting to ubuntu@trial048.front.sepia.ceph.com 2026-02-22T21:38:59.306 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial048.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:38:59.369 DEBUG:teuthology.task.internal:connecting to ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:38:59.369 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial104.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:38:59.434 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-22T21:38:59.446 DEBUG:teuthology.orchestra.run.trial048:> uname -m 2026-02-22T21:38:59.462 INFO:teuthology.orchestra.run.trial048.stdout:x86_64 2026-02-22T21:38:59.462 DEBUG:teuthology.orchestra.run.trial048:> cat /etc/os-release 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:NAME="CentOS Stream" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:VERSION="9" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:ID="centos" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:ID_LIKE="rhel fedora" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:VERSION_ID="9" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:PLATFORM_ID="platform:el9" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:ANSI_COLOR="0;31" 2026-02-22T21:38:59.519 INFO:teuthology.orchestra.run.trial048.stdout:LOGO="fedora-logo-icon" 2026-02-22T21:38:59.520 INFO:teuthology.orchestra.run.trial048.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T21:38:59.520 INFO:teuthology.orchestra.run.trial048.stdout:HOME_URL="https://centos.org/" 2026-02-22T21:38:59.520 INFO:teuthology.orchestra.run.trial048.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T21:38:59.520 INFO:teuthology.orchestra.run.trial048.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T21:38:59.520 INFO:teuthology.orchestra.run.trial048.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T21:38:59.520 INFO:teuthology.lock.ops:Updating trial048.front.sepia.ceph.com on lock server 2026-02-22T21:38:59.590 DEBUG:teuthology.orchestra.run.trial104:> uname -m 2026-02-22T21:38:59.606 INFO:teuthology.orchestra.run.trial104.stdout:x86_64 2026-02-22T21:38:59.606 DEBUG:teuthology.orchestra.run.trial104:> cat /etc/os-release 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:NAME="CentOS Stream" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:VERSION="9" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:ID="centos" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:ID_LIKE="rhel fedora" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:VERSION_ID="9" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:PLATFORM_ID="platform:el9" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:ANSI_COLOR="0;31" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:LOGO="fedora-logo-icon" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:HOME_URL="https://centos.org/" 2026-02-22T21:38:59.661 INFO:teuthology.orchestra.run.trial104.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T21:38:59.662 INFO:teuthology.orchestra.run.trial104.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T21:38:59.662 INFO:teuthology.orchestra.run.trial104.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T21:38:59.662 INFO:teuthology.lock.ops:Updating trial104.front.sepia.ceph.com on lock server 2026-02-22T21:38:59.738 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-22T21:38:59.751 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-22T21:38:59.762 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-22T21:38:59.762 DEBUG:teuthology.orchestra.run.trial048:> test '!' -e /home/ubuntu/cephtest 2026-02-22T21:38:59.764 DEBUG:teuthology.orchestra.run.trial104:> test '!' -e /home/ubuntu/cephtest 2026-02-22T21:38:59.779 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-22T21:38:59.791 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-22T21:38:59.791 DEBUG:teuthology.orchestra.run.trial048:> test -z $(ls -A /var/lib/ceph) 2026-02-22T21:38:59.820 DEBUG:teuthology.orchestra.run.trial104:> test -z $(ls -A /var/lib/ceph) 2026-02-22T21:38:59.837 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-22T21:39:00.001 INFO:teuthology.run_tasks:Running task kernel... 2026-02-22T21:39:00.021 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-22T21:39:00.021 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-22T21:39:00.021 DEBUG:teuthology.orchestra.run.trial048:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T21:39:00.021 DEBUG:teuthology.orchestra.run.trial104:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T21:39:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:39:00.036 DEBUG:teuthology.orchestra.run.trial048:> uname -r 2026-02-22T21:39:00.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:39:00.037 DEBUG:teuthology.orchestra.run.trial104:> uname -r 2026-02-22T21:39:00.092 INFO:teuthology.orchestra.run.trial104.stdout:5.14.0-681.el9.x86_64 2026-02-22T21:39:00.092 INFO:teuthology.task.kernel:Running kernel on trial104: 5.14.0-681.el9.x86_64 2026-02-22T21:39:00.092 DEBUG:teuthology.orchestra.run.trial104:> sudo yum install -y kernel 2026-02-22T21:39:00.092 INFO:teuthology.orchestra.run.trial048.stdout:5.14.0-681.el9.x86_64 2026-02-22T21:39:00.093 INFO:teuthology.task.kernel:Running kernel on trial048: 5.14.0-681.el9.x86_64 2026-02-22T21:39:00.093 DEBUG:teuthology.orchestra.run.trial048:> sudo yum install -y kernel 2026-02-22T21:39:02.142 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - BaseOS 5.3 MB/s | 8.9 MB 00:01 2026-02-22T21:39:05.215 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - AppStream 11 MB/s | 27 MB 00:02 2026-02-22T21:39:13.605 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - CRB 1.3 MB/s | 7.8 MB 00:05 2026-02-22T21:39:14.466 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - Extras packages 148 kB/s | 20 kB 00:00 2026-02-22T21:39:15.695 INFO:teuthology.orchestra.run.trial104.stdout:Extra Packages for Enterprise Linux 17 MB/s | 20 MB 00:01 2026-02-22T21:39:18.867 INFO:teuthology.orchestra.run.trial104.stdout:lab-extras 2.7 MB/s | 50 kB 00:00 2026-02-22T21:39:19.879 INFO:teuthology.orchestra.run.trial104.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T21:39:19.879 INFO:teuthology.orchestra.run.trial104.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T21:39:19.898 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:39:19.899 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:39:19.899 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:39:19.922 DEBUG:teuthology.orchestra.run.trial104:> echo no | sudo yum reinstall kernel || true 2026-02-22T21:39:20.223 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 09:39:18 PM UTC. 2026-02-22T21:39:20.303 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:39:20.305 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:39:20.305 INFO:teuthology.orchestra.run.trial104.stdout: Package Architecture Version Repository Size 2026-02-22T21:39:20.305 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:39:20.305 INFO:teuthology.orchestra.run.trial104.stdout:Reinstalling: 2026-02-22T21:39:20.305 INFO:teuthology.orchestra.run.trial104.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:39:20.306 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:39:20.306 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:39:20.306 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:39:20.306 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:39:20.306 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 803 k 2026-02-22T21:39:20.306 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 0 2026-02-22T21:39:20.307 INFO:teuthology.orchestra.run.trial104.stderr:Operation aborted. 2026-02-22T21:39:20.324 INFO:teuthology.orchestra.run.trial104.stdout:Is this ok [y/N]: 2026-02-22T21:39:20.325 DEBUG:teuthology.orchestra.run.trial104:> sudo yum reinstall -y kernel || true 2026-02-22T21:39:20.627 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 09:39:18 PM UTC. 2026-02-22T21:39:20.707 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout: Package Architecture Version Repository Size 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:Reinstalling: 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 803 k 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 0 2026-02-22T21:39:20.708 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:39:21.413 INFO:teuthology.orchestra.run.trial104.stdout:kernel-5.14.0-681.el9.x86_64.rpm 1.2 MB/s | 803 kB 00:00 2026-02-22T21:39:21.413 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:39:21.413 INFO:teuthology.orchestra.run.trial104.stdout:Total 1.1 MB/s | 803 kB 00:00 2026-02-22T21:39:21.423 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:39:21.456 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:39:21.456 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:39:21.460 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:39:21.461 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:39:21.484 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:39:21.488 INFO:teuthology.orchestra.run.trial104.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:39:21.489 INFO:teuthology.orchestra.run.trial104.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:39:21.601 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:39:21.602 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:39:21.658 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:39:21.658 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:39:21.658 INFO:teuthology.orchestra.run.trial104.stdout:Reinstalled: 2026-02-22T21:39:21.658 INFO:teuthology.orchestra.run.trial104.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T21:39:21.658 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:39:21.658 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:39:21.686 DEBUG:teuthology.orchestra.run.trial104:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T21:39:21.749 INFO:teuthology.orchestra.run.trial104.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T21:39:21.750 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T21:39:21.751 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T21:39:21.751 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-22T21:39:21.751 DEBUG:teuthology.orchestra.run.trial104:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T21:39:21.814 INFO:teuthology.orchestra.run.trial104.stdout:ttyS1 2026-02-22T21:39:21.842 DEBUG:teuthology.parallel:result is None 2026-02-22T21:39:55.188 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - BaseOS 166 kB/s | 8.9 MB 00:54 2026-02-22T21:39:58.871 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - AppStream 8.5 MB/s | 27 MB 00:03 2026-02-22T21:40:10.125 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - CRB 892 kB/s | 7.8 MB 00:08 2026-02-22T21:40:11.884 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - Extras packages 19 kB/s | 20 kB 00:01 2026-02-22T21:40:12.613 INFO:teuthology.orchestra.run.trial048.stdout:Extra Packages for Enterprise Linux 30 MB/s | 20 MB 00:00 2026-02-22T21:40:15.770 INFO:teuthology.orchestra.run.trial048.stdout:lab-extras 2.7 MB/s | 50 kB 00:00 2026-02-22T21:40:16.779 INFO:teuthology.orchestra.run.trial048.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T21:40:16.780 INFO:teuthology.orchestra.run.trial048.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T21:40:16.798 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:40:16.799 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:40:16.799 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:40:16.821 DEBUG:teuthology.orchestra.run.trial048:> echo no | sudo yum reinstall kernel || true 2026-02-22T21:40:17.128 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 09:40:15 PM UTC. 2026-02-22T21:40:17.209 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:40:17.210 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:40:17.210 INFO:teuthology.orchestra.run.trial048.stdout: Package Architecture Version Repository Size 2026-02-22T21:40:17.210 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:40:17.210 INFO:teuthology.orchestra.run.trial048.stdout:Reinstalling: 2026-02-22T21:40:17.210 INFO:teuthology.orchestra.run.trial048.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:40:17.210 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:40:17.210 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:40:17.211 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:40:17.211 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:40:17.211 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 803 k 2026-02-22T21:40:17.211 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 0 2026-02-22T21:40:17.211 INFO:teuthology.orchestra.run.trial048.stderr:Operation aborted. 2026-02-22T21:40:17.230 INFO:teuthology.orchestra.run.trial048.stdout:Is this ok [y/N]: 2026-02-22T21:40:17.231 DEBUG:teuthology.orchestra.run.trial048:> sudo yum reinstall -y kernel || true 2026-02-22T21:40:17.535 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 09:40:15 PM UTC. 2026-02-22T21:40:17.615 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:40:17.615 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:40:17.615 INFO:teuthology.orchestra.run.trial048.stdout: Package Architecture Version Repository Size 2026-02-22T21:40:17.615 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:40:17.615 INFO:teuthology.orchestra.run.trial048.stdout:Reinstalling: 2026-02-22T21:40:17.615 INFO:teuthology.orchestra.run.trial048.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T21:40:17.615 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:40:17.616 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:40:17.616 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:40:17.616 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:40:17.616 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 803 k 2026-02-22T21:40:17.616 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 0 2026-02-22T21:40:17.616 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:40:17.787 INFO:teuthology.orchestra.run.trial048.stdout:kernel-5.14.0-681.el9.x86_64.rpm 10 MB/s | 803 kB 00:00 2026-02-22T21:40:17.787 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:40:17.788 INFO:teuthology.orchestra.run.trial048.stdout:Total 4.6 MB/s | 803 kB 00:00 2026-02-22T21:40:17.797 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:40:17.830 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:40:17.830 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:40:17.835 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:40:17.835 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:40:17.860 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:40:17.864 INFO:teuthology.orchestra.run.trial048.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:40:17.864 INFO:teuthology.orchestra.run.trial048.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:40:17.980 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:40:17.981 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T21:40:18.031 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T21:40:18.032 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:40:18.032 INFO:teuthology.orchestra.run.trial048.stdout:Reinstalled: 2026-02-22T21:40:18.032 INFO:teuthology.orchestra.run.trial048.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T21:40:18.032 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:40:18.032 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:40:18.058 DEBUG:teuthology.orchestra.run.trial048:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T21:40:18.083 INFO:teuthology.orchestra.run.trial048.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T21:40:18.083 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T21:40:18.084 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T21:40:18.084 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-22T21:40:18.084 DEBUG:teuthology.orchestra.run.trial048:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T21:40:18.149 INFO:teuthology.orchestra.run.trial048.stdout:ttyS1 2026-02-22T21:40:18.170 DEBUG:teuthology.parallel:result is None 2026-02-22T21:40:18.170 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-22T21:40:18.182 INFO:teuthology.task.internal:Creating test directory... 2026-02-22T21:40:18.182 DEBUG:teuthology.orchestra.run.trial048:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T21:40:18.212 DEBUG:teuthology.orchestra.run.trial104:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T21:40:18.229 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-22T21:40:18.240 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-22T21:40:18.254 INFO:teuthology.task.internal:Creating archive directory... 2026-02-22T21:40:18.254 DEBUG:teuthology.orchestra.run.trial048:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T21:40:18.269 DEBUG:teuthology.orchestra.run.trial104:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T21:40:18.297 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-22T21:40:18.308 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-22T21:40:18.308 DEBUG:teuthology.orchestra.run.trial048:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T21:40:18.326 DEBUG:teuthology.orchestra.run.trial104:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T21:40:18.348 INFO:teuthology.orchestra.run.trial048.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:40:18.352 INFO:teuthology.orchestra.run.trial104.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:40:18.357 INFO:teuthology.orchestra.run.trial048.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:40:18.361 INFO:teuthology.orchestra.run.trial104.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T21:40:18.363 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-22T21:40:18.375 INFO:teuthology.task.internal:Configuring sudo... 2026-02-22T21:40:18.375 DEBUG:teuthology.orchestra.run.trial048:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T21:40:18.401 DEBUG:teuthology.orchestra.run.trial104:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T21:40:18.426 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-22T21:40:18.443 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-22T21:40:18.443 DEBUG:teuthology.orchestra.run.trial048:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T21:40:18.467 DEBUG:teuthology.orchestra.run.trial104:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T21:40:18.484 DEBUG:teuthology.orchestra.run.trial048:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:40:18.706 DEBUG:teuthology.orchestra.run.trial048:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:40:18.731 DEBUG:teuthology.orchestra.run.trial048:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:40:18.887 DEBUG:teuthology.orchestra.run.trial048:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:40:18.912 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:40:18.912 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T21:40:18.976 DEBUG:teuthology.orchestra.run.trial104:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:40:19.074 DEBUG:teuthology.orchestra.run.trial104:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T21:40:19.100 DEBUG:teuthology.orchestra.run.trial104:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:40:19.229 DEBUG:teuthology.orchestra.run.trial104:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T21:40:19.254 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:40:19.254 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T21:40:19.319 DEBUG:teuthology.orchestra.run.trial048:> sudo service rsyslog restart 2026-02-22T21:40:19.321 DEBUG:teuthology.orchestra.run.trial104:> sudo service rsyslog restart 2026-02-22T21:40:19.348 INFO:teuthology.orchestra.run.trial048.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T21:40:19.388 INFO:teuthology.orchestra.run.trial104.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T21:40:19.764 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-22T21:40:19.777 INFO:teuthology.task.internal:Starting timer... 2026-02-22T21:40:19.777 INFO:teuthology.run_tasks:Running task pcp... 2026-02-22T21:40:19.800 INFO:teuthology.run_tasks:Running task selinux... 2026-02-22T21:40:19.819 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:getty_t:s0']} 2026-02-22T21:40:19.819 DEBUG:teuthology.orchestra.run.trial048:> sudo service auditd rotate 2026-02-22T21:40:19.850 INFO:teuthology.orchestra.run.trial048.stdout:Rotating logs: 2026-02-22T21:40:19.853 DEBUG:teuthology.orchestra.run.trial104:> sudo service auditd rotate 2026-02-22T21:40:19.883 INFO:teuthology.orchestra.run.trial104.stdout:Rotating logs: 2026-02-22T21:40:19.884 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-22T21:40:19.885 DEBUG:teuthology.orchestra.run.trial048:> /usr/sbin/getenforce 2026-02-22T21:40:19.908 INFO:teuthology.orchestra.run.trial048.stdout:Permissive 2026-02-22T21:40:19.909 DEBUG:teuthology.orchestra.run.trial104:> /usr/sbin/getenforce 2026-02-22T21:40:19.942 INFO:teuthology.orchestra.run.trial104.stdout:Permissive 2026-02-22T21:40:19.942 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial048.front.sepia.ceph.com': 'permissive', 'ubuntu@trial104.front.sepia.ceph.com': 'permissive'} 2026-02-22T21:40:19.942 DEBUG:teuthology.orchestra.run.trial048:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T21:40:19.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:40:19.976 DEBUG:teuthology.orchestra.run.trial104:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T21:40:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:40:20.007 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-22T21:40:20.007 DEBUG:teuthology.orchestra.run.trial048:> sudo /usr/sbin/setenforce permissive 2026-02-22T21:40:20.043 DEBUG:teuthology.orchestra.run.trial104:> sudo /usr/sbin/setenforce permissive 2026-02-22T21:40:20.070 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-22T21:40:20.086 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-22T21:40:20.097 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-22T21:40:20.098 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial048.front.sepia.ceph.com,trial104.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-22T21:43:00.710 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial048.front.sepia.ceph.com'), Remote(name='ubuntu@trial104.front.sepia.ceph.com')] 2026-02-22T21:43:00.711 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial048.front.sepia.ceph.com' 2026-02-22T21:43:00.711 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial048.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:43:00.823 DEBUG:teuthology.orchestra.run.trial048:> true 2026-02-22T21:43:00.902 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial048.front.sepia.ceph.com' 2026-02-22T21:43:00.903 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial104.front.sepia.ceph.com' 2026-02-22T21:43:00.904 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial104.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T21:43:01.000 DEBUG:teuthology.orchestra.run.trial104:> true 2026-02-22T21:43:01.075 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial104.front.sepia.ceph.com' 2026-02-22T21:43:01.075 INFO:teuthology.run_tasks:Running task clock... 2026-02-22T21:43:01.088 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-22T21:43:01.088 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T21:43:01.088 DEBUG:teuthology.orchestra.run.trial048:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T21:43:01.090 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T21:43:01.090 DEBUG:teuthology.orchestra.run.trial104:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T21:43:01.116 INFO:teuthology.orchestra.run.trial048.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T21:43:01.128 INFO:teuthology.orchestra.run.trial048.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T21:43:01.143 INFO:teuthology.orchestra.run.trial104.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T21:43:01.149 INFO:teuthology.orchestra.run.trial048.stderr:sudo: ntpd: command not found 2026-02-22T21:43:01.155 INFO:teuthology.orchestra.run.trial104.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T21:43:01.158 INFO:teuthology.orchestra.run.trial048.stdout:506 Cannot talk to daemon 2026-02-22T21:43:01.171 INFO:teuthology.orchestra.run.trial048.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T21:43:01.175 INFO:teuthology.orchestra.run.trial104.stderr:sudo: ntpd: command not found 2026-02-22T21:43:01.183 INFO:teuthology.orchestra.run.trial048.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T21:43:01.185 INFO:teuthology.orchestra.run.trial104.stdout:506 Cannot talk to daemon 2026-02-22T21:43:01.197 INFO:teuthology.orchestra.run.trial104.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T21:43:01.209 INFO:teuthology.orchestra.run.trial104.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T21:43:01.227 INFO:teuthology.orchestra.run.trial048.stderr:bash: line 1: ntpq: command not found 2026-02-22T21:43:01.251 INFO:teuthology.orchestra.run.trial048.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T21:43:01.251 INFO:teuthology.orchestra.run.trial048.stdout:=============================================================================== 2026-02-22T21:43:01.251 INFO:teuthology.orchestra.run.trial048.stdout:^? us-central-1.clearnet.pw 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.252 INFO:teuthology.orchestra.run.trial048.stdout:^? 172-238-164-57.ip.linode> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.252 INFO:teuthology.orchestra.run.trial048.stdout:^? s2-a.time.mci1.us.rozint> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.252 INFO:teuthology.orchestra.run.trial048.stdout:^? LAX.CALTICK.NET 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.258 INFO:teuthology.orchestra.run.trial104.stderr:bash: line 1: ntpq: command not found 2026-02-22T21:43:01.447 INFO:teuthology.orchestra.run.trial104.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T21:43:01.448 INFO:teuthology.orchestra.run.trial104.stdout:=============================================================================== 2026-02-22T21:43:01.448 INFO:teuthology.orchestra.run.trial104.stdout:^? ntp6.kernfusion.at 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.448 INFO:teuthology.orchestra.run.trial104.stdout:^? 170-187-147-56.ip.linode> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.448 INFO:teuthology.orchestra.run.trial104.stdout:^? 38.45.64.130 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.448 INFO:teuthology.orchestra.run.trial104.stdout:^? as393746.mci.trtnw.net 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T21:43:01.449 INFO:teuthology.run_tasks:Running task install... 2026-02-22T21:43:01.463 DEBUG:teuthology.task.install:project ceph 2026-02-22T21:43:01.463 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}} 2026-02-22T21:43:01.464 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'} 2026-02-22T21:43:01.464 INFO:teuthology.task.install:Using flavor: default 2026-02-22T21:43:01.472 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-22T21:43:01.472 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-22T21:43:01.473 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T21:43:01.473 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:43:01.476 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T21:43:01.476 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:43:01.679 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T21:43:01.679 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T21:43:01.691 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T21:43:01.691 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T21:43:01.841 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T21:43:01.841 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:43:01.841 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T21:43:01.852 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T21:43:01.852 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:43:01.852 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T21:43:01.866 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T21:43:01.866 DEBUG:teuthology.orchestra.run.trial048:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T21:43:01.877 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T21:43:01.878 DEBUG:teuthology.orchestra.run.trial104:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T21:43:01.931 DEBUG:teuthology.orchestra.run.trial048:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T21:43:01.942 DEBUG:teuthology.orchestra.run.trial104:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T21:43:02.005 DEBUG:teuthology.orchestra.run.trial048:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T21:43:02.017 DEBUG:teuthology.orchestra.run.trial104:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T21:43:02.068 INFO:teuthology.orchestra.run.trial048.stdout:check_obsoletes = 1 2026-02-22T21:43:02.070 DEBUG:teuthology.orchestra.run.trial048:> sudo yum clean all 2026-02-22T21:43:02.079 INFO:teuthology.orchestra.run.trial104.stdout:check_obsoletes = 1 2026-02-22T21:43:02.081 DEBUG:teuthology.orchestra.run.trial104:> sudo yum clean all 2026-02-22T21:43:02.226 INFO:teuthology.orchestra.run.trial048.stdout:41 files removed 2026-02-22T21:43:02.237 INFO:teuthology.orchestra.run.trial104.stdout:41 files removed 2026-02-22T21:43:02.241 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-radosgw 2026-02-22T21:43:02.252 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-radosgw 2026-02-22T21:43:02.520 INFO:teuthology.orchestra.run.trial048.stdout:ceph packages for x86_64 618 kB/s | 86 kB 00:00 2026-02-22T21:43:02.523 INFO:teuthology.orchestra.run.trial104.stdout:ceph packages for x86_64 661 kB/s | 86 kB 00:00 2026-02-22T21:43:02.652 INFO:teuthology.orchestra.run.trial048.stdout:ceph noarch packages 112 kB/s | 12 kB 00:00 2026-02-22T21:43:02.653 INFO:teuthology.orchestra.run.trial104.stdout:ceph noarch packages 114 kB/s | 12 kB 00:00 2026-02-22T21:43:02.779 INFO:teuthology.orchestra.run.trial048.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T21:43:02.781 INFO:teuthology.orchestra.run.trial104.stdout:ceph source packages 17 kB/s | 1.9 kB 00:00 2026-02-22T21:43:03.717 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - BaseOS 9.7 MB/s | 8.9 MB 00:00 2026-02-22T21:43:03.757 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - BaseOS 9.3 MB/s | 8.9 MB 00:00 2026-02-22T21:43:07.488 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - AppStream 8.4 MB/s | 27 MB 00:03 2026-02-22T21:43:07.654 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - AppStream 7.9 MB/s | 27 MB 00:03 2026-02-22T21:43:11.544 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - CRB 4.6 MB/s | 7.8 MB 00:01 2026-02-22T21:43:12.398 INFO:teuthology.orchestra.run.trial104.stdout:CentOS Stream 9 - Extras packages 162 kB/s | 20 kB 00:00 2026-02-22T21:43:12.938 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - CRB 2.7 MB/s | 7.8 MB 00:02 2026-02-22T21:43:13.765 INFO:teuthology.orchestra.run.trial104.stdout:Extra Packages for Enterprise Linux 16 MB/s | 20 MB 00:01 2026-02-22T21:43:14.102 INFO:teuthology.orchestra.run.trial048.stdout:CentOS Stream 9 - Extras packages 47 kB/s | 20 kB 00:00 2026-02-22T21:43:14.991 INFO:teuthology.orchestra.run.trial048.stdout:Extra Packages for Enterprise Linux 25 MB/s | 20 MB 00:00 2026-02-22T21:43:16.974 INFO:teuthology.orchestra.run.trial104.stdout:lab-extras 2.0 MB/s | 50 kB 00:00 2026-02-22T21:43:18.178 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:18.179 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout: Package Arch Version Repository Size 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout:Upgrading: 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout:Installing dependencies: 2026-02-22T21:43:18.180 INFO:teuthology.orchestra.run.trial104.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T21:43:18.181 INFO:teuthology.orchestra.run.trial104.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T21:43:18.182 INFO:teuthology.orchestra.run.trial104.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout:Installing weak dependencies: 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout:Install 33 Packages 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout:Upgrade 2 Packages 2026-02-22T21:43:18.183 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:18.184 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 63 M 2026-02-22T21:43:18.184 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:43:18.185 INFO:teuthology.orchestra.run.trial048.stdout:lab-extras 2.6 MB/s | 50 kB 00:00 2026-02-22T21:43:18.736 INFO:teuthology.orchestra.run.trial104.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 16 MB/s | 5.5 MB 00:00 2026-02-22T21:43:18.769 INFO:teuthology.orchestra.run.trial104.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 763 kB/s | 25 kB 00:00 2026-02-22T21:43:18.827 INFO:teuthology.orchestra.run.trial104.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 17 MB/s | 999 kB 00:00 2026-02-22T21:43:18.876 INFO:teuthology.orchestra.run.trial104.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 9.9 MB/s | 503 kB 00:00 2026-02-22T21:43:19.138 INFO:teuthology.orchestra.run.trial104.stdout:(5/35): ceph-common-19.2.3-689.gf90f4eae.el9.x8 29 MB/s | 22 MB 00:00 2026-02-22T21:43:19.149 INFO:teuthology.orchestra.run.trial104.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 3.8 MB/s | 45 kB 00:00 2026-02-22T21:43:19.162 INFO:teuthology.orchestra.run.trial104.stdout:(7/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_64 19 MB/s | 5.4 MB 00:00 2026-02-22T21:43:19.165 INFO:teuthology.orchestra.run.trial104.stdout:(8/35): python3-ceph-common-19.2.3-689.gf90f4ea 9.0 MB/s | 142 kB 00:00 2026-02-22T21:43:19.176 INFO:teuthology.orchestra.run.trial104.stdout:(9/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 11 MB/s | 165 kB 00:00 2026-02-22T21:43:19.184 INFO:teuthology.orchestra.run.trial104.stdout:(10/35): python3-rados-19.2.3-689.gf90f4eae.el9 17 MB/s | 322 kB 00:00 2026-02-22T21:43:19.198 INFO:teuthology.orchestra.run.trial104.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 14 MB/s | 303 kB 00:00 2026-02-22T21:43:19.201 INFO:teuthology.orchestra.run.trial104.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 5.7 MB/s | 100 kB 00:00 2026-02-22T21:43:19.258 INFO:teuthology.orchestra.run.trial104.stdout:(13/35): mailcap-2.1.49-5.el9.noarch.rpm 552 kB/s | 33 kB 00:00 2026-02-22T21:43:19.280 INFO:teuthology.orchestra.run.trial104.stdout:(14/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9. 12 MB/s | 11 MB 00:00 2026-02-22T21:43:19.300 INFO:teuthology.orchestra.run.trial104.stdout:(15/35): pciutils-3.7.0-7.el9.x86_64.rpm 2.2 MB/s | 93 kB 00:00 2026-02-22T21:43:19.359 INFO:teuthology.orchestra.run.trial104.stdout:(16/35): nvme-cli-2.16-1.el9.x86_64.rpm 7.3 MB/s | 1.2 MB 00:00 2026-02-22T21:43:19.384 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:43:19.385 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout: Package Arch Version Repository Size 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout:Upgrading: 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout:Installing dependencies: 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T21:43:19.386 INFO:teuthology.orchestra.run.trial048.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T21:43:19.387 INFO:teuthology.orchestra.run.trial048.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T21:43:19.388 INFO:teuthology.orchestra.run.trial048.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout:Installing weak dependencies: 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout:Install 33 Packages 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout:Upgrade 2 Packages 2026-02-22T21:43:19.389 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:19.390 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 63 M 2026-02-22T21:43:19.390 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:43:19.409 INFO:teuthology.orchestra.run.trial104.stdout:(17/35): boost-program-options-1.75.0-13.el9.x8 801 kB/s | 104 kB 00:00 2026-02-22T21:43:19.441 INFO:teuthology.orchestra.run.trial104.stdout:(18/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 1.4 MB/s | 45 kB 00:00 2026-02-22T21:43:19.442 INFO:teuthology.orchestra.run.trial104.stdout:(19/35): libnbd-1.20.3-4.el9.x86_64.rpm 1.1 MB/s | 164 kB 00:00 2026-02-22T21:43:19.498 INFO:teuthology.orchestra.run.trial104.stdout:(20/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.1 MB/s | 160 kB 00:00 2026-02-22T21:43:19.501 INFO:teuthology.orchestra.run.trial104.stdout:(21/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 4.8 MB/s | 292 kB 00:00 2026-02-22T21:43:19.527 INFO:teuthology.orchestra.run.trial104.stdout:(22/35): qatlib-service-25.08.0-2.el9.x86_64.rp 1.4 MB/s | 37 kB 00:00 2026-02-22T21:43:19.534 INFO:teuthology.orchestra.run.trial104.stdout:(23/35): librdkafka-1.6.1-102.el9.x86_64.rpm 6.9 MB/s | 662 kB 00:00 2026-02-22T21:43:19.554 INFO:teuthology.orchestra.run.trial104.stdout:(24/35): qatlib-25.08.0-2.el9.x86_64.rpm 4.2 MB/s | 240 kB 00:00 2026-02-22T21:43:19.555 INFO:teuthology.orchestra.run.trial104.stdout:(25/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 2.4 MB/s | 66 kB 00:00 2026-02-22T21:43:19.766 INFO:teuthology.orchestra.run.trial104.stdout:(26/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 117 kB/s | 25 kB 00:00 2026-02-22T21:43:19.987 INFO:teuthology.orchestra.run.trial104.stdout:(27/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 680 kB/s | 308 kB 00:00 2026-02-22T21:43:20.044 INFO:teuthology.orchestra.run.trial104.stdout:(28/35): liboath-2.6.12-1.el9.x86_64.rpm 176 kB/s | 49 kB 00:00 2026-02-22T21:43:20.070 INFO:teuthology.orchestra.run.trial048.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 19 MB/s | 5.5 MB 00:00 2026-02-22T21:43:20.100 INFO:teuthology.orchestra.run.trial048.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 842 kB/s | 25 kB 00:00 2026-02-22T21:43:20.153 INFO:teuthology.orchestra.run.trial048.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 18 MB/s | 999 kB 00:00 2026-02-22T21:43:20.200 INFO:teuthology.orchestra.run.trial048.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 11 MB/s | 503 kB 00:00 2026-02-22T21:43:20.273 INFO:teuthology.orchestra.run.trial104.stdout:(29/35): libunwind-1.6.2-1.el9.x86_64.rpm 236 kB/s | 67 kB 00:00 2026-02-22T21:43:20.486 INFO:teuthology.orchestra.run.trial048.stdout:(5/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 15 MB/s | 11 MB 00:00 2026-02-22T21:43:20.496 INFO:teuthology.orchestra.run.trial048.stdout:(6/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_64 18 MB/s | 5.4 MB 00:00 2026-02-22T21:43:20.506 INFO:teuthology.orchestra.run.trial048.stdout:(7/35): python3-ceph-argparse-19.2.3-689.gf90f4 2.2 MB/s | 45 kB 00:00 2026-02-22T21:43:20.519 INFO:teuthology.orchestra.run.trial048.stdout:(8/35): python3-ceph-common-19.2.3-689.gf90f4ea 6.2 MB/s | 142 kB 00:00 2026-02-22T21:43:20.529 INFO:teuthology.orchestra.run.trial048.stdout:(9/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 6.8 MB/s | 165 kB 00:00 2026-02-22T21:43:20.531 INFO:teuthology.orchestra.run.trial104.stdout:(30/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 1.7 MB/s | 838 kB 00:00 2026-02-22T21:43:20.540 INFO:teuthology.orchestra.run.trial104.stdout:(31/35): libarrow-9.0.0-15.el9.x86_64.rpm 4.5 MB/s | 4.4 MB 00:00 2026-02-22T21:43:20.545 INFO:teuthology.orchestra.run.trial048.stdout:(10/35): python3-rados-19.2.3-689.gf90f4eae.el9 12 MB/s | 322 kB 00:00 2026-02-22T21:43:20.559 INFO:teuthology.orchestra.run.trial048.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 10 MB/s | 303 kB 00:00 2026-02-22T21:43:20.572 INFO:teuthology.orchestra.run.trial048.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 3.7 MB/s | 100 kB 00:00 2026-02-22T21:43:20.621 INFO:teuthology.orchestra.run.trial104.stdout:(32/35): re2-20211101-20.el9.x86_64.rpm 550 kB/s | 191 kB 00:00 2026-02-22T21:43:20.702 INFO:teuthology.orchestra.run.trial048.stdout:(13/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 24 MB/s | 22 MB 00:00 2026-02-22T21:43:20.735 INFO:teuthology.orchestra.run.trial104.stdout:(33/35): librados2-19.2.3-689.gf90f4eae.el9.x86 18 MB/s | 3.4 MB 00:00 2026-02-22T21:43:20.784 INFO:teuthology.orchestra.run.trial104.stdout:(34/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 19 MB/s | 3.2 MB 00:00 2026-02-22T21:43:20.963 INFO:teuthology.orchestra.run.trial048.stdout:(14/35): mailcap-2.1.49-5.el9.noarch.rpm 82 kB/s | 33 kB 00:00 2026-02-22T21:43:21.161 INFO:teuthology.orchestra.run.trial104.stdout:(35/35): thrift-0.15.0-4.el9.x86_64.rpm 2.5 MB/s | 1.6 MB 00:00 2026-02-22T21:43:21.164 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:21.165 INFO:teuthology.orchestra.run.trial104.stdout:Total 21 MB/s | 63 MB 00:02 2026-02-22T21:43:21.257 INFO:teuthology.orchestra.run.trial048.stdout:(15/35): pciutils-3.7.0-7.el9.x86_64.rpm 168 kB/s | 93 kB 00:00 2026-02-22T21:43:21.263 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:43:21.281 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:43:21.281 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:43:21.536 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:43:21.537 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:43:21.632 INFO:teuthology.orchestra.run.trial048.stdout:(16/35): boost-program-options-1.75.0-13.el9.x8 156 kB/s | 104 kB 00:00 2026-02-22T21:43:21.831 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:43:21.840 INFO:teuthology.orchestra.run.trial104.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T21:43:21.843 INFO:teuthology.orchestra.run.trial104.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T21:43:21.855 INFO:teuthology.orchestra.run.trial104.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T21:43:21.976 INFO:teuthology.orchestra.run.trial048.stdout:(17/35): libnbd-1.20.3-4.el9.x86_64.rpm 228 kB/s | 164 kB 00:00 2026-02-22T21:43:22.005 INFO:teuthology.orchestra.run.trial104.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T21:43:22.010 INFO:teuthology.orchestra.run.trial104.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:43:22.040 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:43:22.084 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:43:22.085 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:43:22.098 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:43:22.102 INFO:teuthology.orchestra.run.trial104.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T21:43:22.111 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T21:43:22.133 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:43:22.134 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:43:22.147 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:43:22.178 INFO:teuthology.orchestra.run.trial104.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T21:43:22.182 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T21:43:22.188 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T21:43:22.190 INFO:teuthology.orchestra.run.trial104.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T21:43:22.193 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T21:43:22.198 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T21:43:22.338 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:43:22.339 INFO:teuthology.orchestra.run.trial104.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:43:22.350 INFO:teuthology.orchestra.run.trial048.stdout:(18/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 223 kB/s | 160 kB 00:00 2026-02-22T21:43:22.365 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:43:22.367 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T21:43:22.375 INFO:teuthology.orchestra.run.trial104.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T21:43:22.386 INFO:teuthology.orchestra.run.trial048.stdout:(19/35): nvme-cli-2.16-1.el9.x86_64.rpm 650 kB/s | 1.2 MB 00:01 2026-02-22T21:43:22.561 INFO:teuthology.orchestra.run.trial048.stdout:(20/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 77 kB/s | 45 kB 00:00 2026-02-22T21:43:22.601 INFO:teuthology.orchestra.run.trial104.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T21:43:22.602 INFO:teuthology.orchestra.run.trial104.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:43:22.618 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:43:22.622 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T21:43:22.628 INFO:teuthology.orchestra.run.trial104.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T21:43:22.631 INFO:teuthology.orchestra.run.trial104.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T21:43:22.632 INFO:teuthology.orchestra.run.trial104.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:43:22.650 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:43:22.766 INFO:teuthology.orchestra.run.trial104.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T21:43:22.768 INFO:teuthology.orchestra.run.trial104.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:43:23.052 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:43:23.052 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T21:43:23.052 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:23.089 INFO:teuthology.orchestra.run.trial104.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T21:43:23.090 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T21:43:23.171 INFO:teuthology.orchestra.run.trial048.stdout:(21/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 372 kB/s | 292 kB 00:00 2026-02-22T21:43:23.222 INFO:teuthology.orchestra.run.trial048.stdout:(22/35): librdkafka-1.6.1-102.el9.x86_64.rpm 760 kB/s | 662 kB 00:00 2026-02-22T21:43:23.322 INFO:teuthology.orchestra.run.trial048.stdout:(23/35): qatlib-25.08.0-2.el9.x86_64.rpm 315 kB/s | 240 kB 00:00 2026-02-22T21:43:23.531 INFO:teuthology.orchestra.run.trial048.stdout:(24/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.4 MB/s | 308 kB 00:00 2026-02-22T21:43:23.672 INFO:teuthology.orchestra.run.trial048.stdout:(25/35): libarrow-9.0.0-15.el9.x86_64.rpm 31 MB/s | 4.4 MB 00:00 2026-02-22T21:43:23.699 INFO:teuthology.orchestra.run.trial048.stdout:(26/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 928 kB/s | 25 kB 00:00 2026-02-22T21:43:23.726 INFO:teuthology.orchestra.run.trial048.stdout:(27/35): liboath-2.6.12-1.el9.x86_64.rpm 1.8 MB/s | 49 kB 00:00 2026-02-22T21:43:23.742 INFO:teuthology.orchestra.run.trial048.stdout:(28/35): qatlib-service-25.08.0-2.el9.x86_64.rp 65 kB/s | 37 kB 00:00 2026-02-22T21:43:23.753 INFO:teuthology.orchestra.run.trial048.stdout:(29/35): libunwind-1.6.2-1.el9.x86_64.rpm 2.4 MB/s | 67 kB 00:00 2026-02-22T21:43:23.781 INFO:teuthology.orchestra.run.trial048.stdout:(30/35): re2-20211101-20.el9.x86_64.rpm 6.8 MB/s | 191 kB 00:00 2026-02-22T21:43:23.816 INFO:teuthology.orchestra.run.trial048.stdout:(31/35): thrift-0.15.0-4.el9.x86_64.rpm 46 MB/s | 1.6 MB 00:00 2026-02-22T21:43:23.845 INFO:teuthology.orchestra.run.trial048.stdout:(32/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 106 kB/s | 66 kB 00:00 2026-02-22T21:43:23.948 INFO:teuthology.orchestra.run.trial048.stdout:(33/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 4.0 MB/s | 838 kB 00:00 2026-02-22T21:43:24.050 INFO:teuthology.orchestra.run.trial048.stdout:(34/35): librados2-19.2.3-689.gf90f4eae.el9.x86 15 MB/s | 3.4 MB 00:00 2026-02-22T21:43:24.075 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:43:24.077 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:43:24.135 INFO:teuthology.orchestra.run.trial048.stdout:(35/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 11 MB/s | 3.2 MB 00:00 2026-02-22T21:43:24.139 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:24.139 INFO:teuthology.orchestra.run.trial048.stdout:Total 13 MB/s | 63 MB 00:04 2026-02-22T21:43:24.237 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:43:24.255 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:43:24.256 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:43:24.358 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:43:24.361 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:43:24.400 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:43:24.400 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T21:43:24.400 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T21:43:24.400 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:24.404 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:43:24.512 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:43:24.512 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:43:24.806 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:43:24.815 INFO:teuthology.orchestra.run.trial048.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T21:43:24.818 INFO:teuthology.orchestra.run.trial048.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T21:43:24.829 INFO:teuthology.orchestra.run.trial048.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T21:43:24.980 INFO:teuthology.orchestra.run.trial048.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T21:43:24.984 INFO:teuthology.orchestra.run.trial048.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:43:25.013 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:43:25.056 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:43:25.057 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:43:25.067 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:43:25.071 INFO:teuthology.orchestra.run.trial048.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T21:43:25.080 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T21:43:25.102 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:43:25.103 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:43:25.115 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:43:25.147 INFO:teuthology.orchestra.run.trial048.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T21:43:25.151 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T21:43:25.158 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T21:43:25.160 INFO:teuthology.orchestra.run.trial048.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T21:43:25.163 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T21:43:25.169 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T21:43:25.308 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:43:25.310 INFO:teuthology.orchestra.run.trial048.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:43:25.337 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T21:43:25.342 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T21:43:25.348 INFO:teuthology.orchestra.run.trial048.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T21:43:25.574 INFO:teuthology.orchestra.run.trial048.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T21:43:25.575 INFO:teuthology.orchestra.run.trial048.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:43:25.592 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T21:43:25.596 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T21:43:25.601 INFO:teuthology.orchestra.run.trial048.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T21:43:25.603 INFO:teuthology.orchestra.run.trial048.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T21:43:25.604 INFO:teuthology.orchestra.run.trial048.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:43:25.623 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T21:43:25.740 INFO:teuthology.orchestra.run.trial048.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T21:43:25.742 INFO:teuthology.orchestra.run.trial048.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:43:26.019 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T21:43:26.019 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T21:43:26.019 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:26.055 INFO:teuthology.orchestra.run.trial048.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T21:43:26.057 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T21:43:27.038 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:43:27.040 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:43:27.324 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T21:43:27.327 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:43:27.366 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T21:43:27.367 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T21:43:27.367 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T21:43:27.367 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:27.370 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:43:28.948 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:43:28.948 INFO:teuthology.orchestra.run.trial104.stdout:skipping the directory /sys 2026-02-22T21:43:28.949 INFO:teuthology.orchestra.run.trial104.stdout:skipping the directory /proc 2026-02-22T21:43:28.949 INFO:teuthology.orchestra.run.trial104.stdout:skipping the directory /mnt 2026-02-22T21:43:28.949 INFO:teuthology.orchestra.run.trial104.stdout:skipping the directory /var/tmp 2026-02-22T21:43:28.949 INFO:teuthology.orchestra.run.trial104.stdout:skipping the directory /home 2026-02-22T21:43:28.949 INFO:teuthology.orchestra.run.trial104.stdout:skipping the directory /root 2026-02-22T21:43:28.949 INFO:teuthology.orchestra.run.trial104.stdout:skipping the directory /tmp 2026-02-22T21:43:28.949 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:29.398 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:43:29.421 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:43:29.421 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:29.422 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T21:43:29.422 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:43:29.422 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:43:29.422 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:29.423 INFO:teuthology.orchestra.run.trial104.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:43:29.436 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:43:29.436 INFO:teuthology.orchestra.run.trial104.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:43:29.491 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:43:29.491 INFO:teuthology.orchestra.run.trial104.stdout:Creating group 'qat' with GID 992. 2026-02-22T21:43:29.491 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:29.772 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T21:43:29.773 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T21:43:29.773 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T21:43:29.773 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T21:43:29.773 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:43:29.773 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:43:29.773 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T21:43:29.774 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T21:43:29.775 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T21:43:29.776 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T21:43:29.833 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:43:29.833 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:29.833 INFO:teuthology.orchestra.run.trial104.stdout:Upgraded: 2026-02-22T21:43:29.833 INFO:teuthology.orchestra.run.trial104.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.833 INFO:teuthology.orchestra.run.trial104.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.833 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:43:29.833 INFO:teuthology.orchestra.run.trial104.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T21:43:29.834 INFO:teuthology.orchestra.run.trial104.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T21:43:29.835 INFO:teuthology.orchestra.run.trial104.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T21:43:29.836 INFO:teuthology.orchestra.run.trial104.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T21:43:29.837 INFO:teuthology.orchestra.run.trial104.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T21:43:29.837 INFO:teuthology.orchestra.run.trial104.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T21:43:29.837 INFO:teuthology.orchestra.run.trial104.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T21:43:29.837 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:29.837 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:29.876 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-test 2026-02-22T21:43:30.192 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:14 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:30.276 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:30.276 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:30.276 INFO:teuthology.orchestra.run.trial104.stdout: Package Arch Version Repository Size 2026-02-22T21:43:30.276 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:30.276 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout:Installing dependencies: 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:43:30.277 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:30.278 INFO:teuthology.orchestra.run.trial104.stdout:Install 5 Packages 2026-02-22T21:43:30.278 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:30.278 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 50 M 2026-02-22T21:43:30.279 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 212 M 2026-02-22T21:43:30.279 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:43:30.668 INFO:teuthology.orchestra.run.trial104.stdout:(1/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.8 MB/s | 163 kB 00:00 2026-02-22T21:43:30.733 INFO:teuthology.orchestra.run.trial104.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 1.5 MB/s | 233 kB 00:00 2026-02-22T21:43:30.762 INFO:teuthology.orchestra.run.trial104.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.2 MB/s | 64 kB 00:00 2026-02-22T21:43:30.831 INFO:teuthology.orchestra.run.trial104.stdout:(4/5): socat-1.7.4.1-8.el9.x86_64.rpm 1.8 MB/s | 303 kB 00:00 2026-02-22T21:43:31.596 INFO:teuthology.orchestra.run.trial104.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 49 MB/s | 50 MB 00:01 2026-02-22T21:43:31.597 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:31.598 INFO:teuthology.orchestra.run.trial104.stdout:Total 38 MB/s | 50 MB 00:01 2026-02-22T21:43:31.624 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:43:31.633 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:43:31.633 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:43:31.819 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:43:31.819 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:43:31.858 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout:skipping the directory /sys 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout:skipping the directory /proc 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout:skipping the directory /mnt 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout:skipping the directory /var/tmp 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout:skipping the directory /home 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout:skipping the directory /root 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout:skipping the directory /tmp 2026-02-22T21:43:31.859 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:32.031 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:43:32.036 INFO:teuthology.orchestra.run.trial104.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T21:43:32.040 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T21:43:32.048 INFO:teuthology.orchestra.run.trial104.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T21:43:32.049 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:43:32.308 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:43:32.331 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T21:43:32.331 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:32.331 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T21:43:32.331 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:43:32.331 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T21:43:32.331 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:32.333 INFO:teuthology.orchestra.run.trial048.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:43:32.345 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T21:43:32.345 INFO:teuthology.orchestra.run.trial048.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:43:32.401 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:43:32.401 INFO:teuthology.orchestra.run.trial048.stdout:Creating group 'qat' with GID 992. 2026-02-22T21:43:32.401 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:32.687 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T21:43:32.688 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T21:43:32.689 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout:Upgraded: 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T21:43:32.751 INFO:teuthology.orchestra.run.trial048.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.752 INFO:teuthology.orchestra.run.trial048.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T21:43:32.753 INFO:teuthology.orchestra.run.trial048.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T21:43:32.754 INFO:teuthology.orchestra.run.trial048.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T21:43:32.754 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:32.754 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:43:32.796 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-test 2026-02-22T21:43:33.103 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:15 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:43:33.186 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:43:33.187 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout: Package Arch Version Repository Size 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout:Installing dependencies: 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T21:43:33.188 INFO:teuthology.orchestra.run.trial048.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout:Install 5 Packages 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 50 M 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 212 M 2026-02-22T21:43:33.189 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:43:33.432 INFO:teuthology.orchestra.run.trial048.stdout:(1/5): libxslt-1.1.34-12.el9.x86_64.rpm 1.5 MB/s | 233 kB 00:00 2026-02-22T21:43:33.490 INFO:teuthology.orchestra.run.trial048.stdout:(2/5): socat-1.7.4.1-8.el9.x86_64.rpm 5.1 MB/s | 303 kB 00:00 2026-02-22T21:43:33.516 INFO:teuthology.orchestra.run.trial048.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.4 MB/s | 64 kB 00:00 2026-02-22T21:43:33.603 INFO:teuthology.orchestra.run.trial048.stdout:(4/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 501 kB/s | 163 kB 00:00 2026-02-22T21:43:34.297 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:43:34.315 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:43:34.539 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:43:34.539 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T21:43:34.539 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T21:43:34.540 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T21:43:34.540 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T21:43:34.592 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T21:43:34.592 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:34.592 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:43:34.592 INFO:teuthology.orchestra.run.trial104.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:34.593 INFO:teuthology.orchestra.run.trial104.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:34.593 INFO:teuthology.orchestra.run.trial104.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T21:43:34.593 INFO:teuthology.orchestra.run.trial104.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T21:43:34.593 INFO:teuthology.orchestra.run.trial104.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T21:43:34.593 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:34.593 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:34.624 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph 2026-02-22T21:43:34.928 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:18 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:35.013 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:35.016 INFO:teuthology.orchestra.run.trial104.stdout:====================================================================================== 2026-02-22T21:43:35.016 INFO:teuthology.orchestra.run.trial104.stdout: Package Arch Version Repository Size 2026-02-22T21:43:35.016 INFO:teuthology.orchestra.run.trial104.stdout:====================================================================================== 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout:Installing dependencies: 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T21:43:35.017 INFO:teuthology.orchestra.run.trial104.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T21:43:35.018 INFO:teuthology.orchestra.run.trial104.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T21:43:35.019 INFO:teuthology.orchestra.run.trial104.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T21:43:35.020 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T21:43:35.021 INFO:teuthology.orchestra.run.trial104.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T21:43:35.022 INFO:teuthology.orchestra.run.trial104.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T21:43:35.023 INFO:teuthology.orchestra.run.trial104.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout:Installing weak dependencies: 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T21:43:35.024 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout:====================================================================================== 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout:Install 99 Packages 2026-02-22T21:43:35.025 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:35.026 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 95 M 2026-02-22T21:43:35.026 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 520 M 2026-02-22T21:43:35.026 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:43:35.927 INFO:teuthology.orchestra.run.trial104.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 36 kB/s | 6.5 kB 00:00 2026-02-22T21:43:35.955 INFO:teuthology.orchestra.run.trial048.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 19 MB/s | 50 MB 00:02 2026-02-22T21:43:35.957 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:35.957 INFO:teuthology.orchestra.run.trial048.stdout:Total 18 MB/s | 50 MB 00:02 2026-02-22T21:43:35.983 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:43:35.991 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:43:35.991 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:43:36.177 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:43:36.177 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:43:36.389 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:43:36.395 INFO:teuthology.orchestra.run.trial048.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T21:43:36.399 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T21:43:36.407 INFO:teuthology.orchestra.run.trial048.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T21:43:36.408 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:43:36.856 INFO:teuthology.orchestra.run.trial104.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 992 kB/s | 1.1 MB 00:01 2026-02-22T21:43:37.468 INFO:teuthology.orchestra.run.trial104.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 1.4 MB/s | 2.4 MB 00:01 2026-02-22T21:43:37.517 INFO:teuthology.orchestra.run.trial104.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 641 kB/s | 31 kB 00:00 2026-02-22T21:43:37.567 INFO:teuthology.orchestra.run.trial104.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 2.9 MB/s | 150 kB 00:00 2026-02-22T21:43:38.215 INFO:teuthology.orchestra.run.trial104.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 2.1 MB/s | 4.7 MB 00:02 2026-02-22T21:43:38.577 INFO:teuthology.orchestra.run.trial104.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 3.8 MB/s | 3.8 MB 00:01 2026-02-22T21:43:38.631 INFO:teuthology.orchestra.run.trial104.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 404 kB/s | 22 kB 00:00 2026-02-22T21:43:38.695 INFO:teuthology.orchestra.run.trial104.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 3.9 MB/s | 253 kB 00:00 2026-02-22T21:43:38.709 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T21:43:38.727 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:43:38.751 INFO:teuthology.orchestra.run.trial104.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 878 kB/s | 49 kB 00:00 2026-02-22T21:43:38.811 INFO:teuthology.orchestra.run.trial104.stdout:(11/99): ceph-prometheus-alerts-19.2.3-689.gf90 279 kB/s | 17 kB 00:00 2026-02-22T21:43:38.877 INFO:teuthology.orchestra.run.trial104.stdout:(12/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 4.4 MB/s | 299 kB 00:00 2026-02-22T21:43:38.978 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T21:43:38.978 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T21:43:38.978 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T21:43:38.978 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T21:43:38.978 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T21:43:39.030 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T21:43:39.030 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:39.030 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:43:39.030 INFO:teuthology.orchestra.run.trial048.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:39.030 INFO:teuthology.orchestra.run.trial048.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:39.030 INFO:teuthology.orchestra.run.trial048.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T21:43:39.031 INFO:teuthology.orchestra.run.trial048.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T21:43:39.031 INFO:teuthology.orchestra.run.trial048.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T21:43:39.031 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:39.031 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:43:39.063 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph 2026-02-22T21:43:39.069 INFO:teuthology.orchestra.run.trial104.stdout:(13/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 3.9 MB/s | 769 kB 00:00 2026-02-22T21:43:39.091 INFO:teuthology.orchestra.run.trial104.stdout:(14/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 16 MB/s | 351 kB 00:00 2026-02-22T21:43:39.096 INFO:teuthology.orchestra.run.trial104.stdout:(15/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 9.0 MB/s | 40 kB 00:00 2026-02-22T21:43:39.100 INFO:teuthology.orchestra.run.trial104.stdout:(16/99): libconfig-1.7.2-9.el9.x86_64.rpm 15 MB/s | 72 kB 00:00 2026-02-22T21:43:39.115 INFO:teuthology.orchestra.run.trial104.stdout:(17/99): libgfortran-11.5.0-14.el9.x86_64.rpm 53 MB/s | 794 kB 00:00 2026-02-22T21:43:39.123 INFO:teuthology.orchestra.run.trial104.stdout:(18/99): libquadmath-11.5.0-14.el9.x86_64.rpm 24 MB/s | 184 kB 00:00 2026-02-22T21:43:39.130 INFO:teuthology.orchestra.run.trial104.stdout:(19/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 36 MB/s | 253 kB 00:00 2026-02-22T21:43:39.170 INFO:teuthology.orchestra.run.trial104.stdout:(20/99): python3-cryptography-36.0.1-5.el9.x86_ 32 MB/s | 1.2 MB 00:00 2026-02-22T21:43:39.175 INFO:teuthology.orchestra.run.trial104.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 18 MB/s | 106 kB 00:00 2026-02-22T21:43:39.182 INFO:teuthology.orchestra.run.trial104.stdout:(22/99): python3-pycparser-2.20-6.el9.noarch.rp 22 MB/s | 135 kB 00:00 2026-02-22T21:43:39.188 INFO:teuthology.orchestra.run.trial104.stdout:(23/99): python3-pyparsing-2.4.7-9.el9.noarch.r 23 MB/s | 150 kB 00:00 2026-02-22T21:43:39.194 INFO:teuthology.orchestra.run.trial104.stdout:(24/99): python3-requests-2.25.1-10.el9.noarch. 19 MB/s | 126 kB 00:00 2026-02-22T21:43:39.201 INFO:teuthology.orchestra.run.trial104.stdout:(25/99): python3-urllib3-1.26.5-7.el9.noarch.rp 33 MB/s | 218 kB 00:00 2026-02-22T21:43:39.208 INFO:teuthology.orchestra.run.trial104.stdout:(26/99): unzip-6.0-59.el9.x86_64.rpm 26 MB/s | 182 kB 00:00 2026-02-22T21:43:39.214 INFO:teuthology.orchestra.run.trial104.stdout:(27/99): zip-3.0-35.el9.x86_64.rpm 43 MB/s | 266 kB 00:00 2026-02-22T21:43:39.363 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:21 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:43:39.450 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout:====================================================================================== 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout: Package Arch Version Repository Size 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout:====================================================================================== 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout:Installing dependencies: 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T21:43:39.453 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T21:43:39.454 INFO:teuthology.orchestra.run.trial048.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T21:43:39.455 INFO:teuthology.orchestra.run.trial048.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T21:43:39.456 INFO:teuthology.orchestra.run.trial048.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T21:43:39.457 INFO:teuthology.orchestra.run.trial048.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T21:43:39.458 INFO:teuthology.orchestra.run.trial048.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T21:43:39.459 INFO:teuthology.orchestra.run.trial048.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T21:43:39.460 INFO:teuthology.orchestra.run.trial048.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout:Installing weak dependencies: 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout:====================================================================================== 2026-02-22T21:43:39.461 INFO:teuthology.orchestra.run.trial048.stdout:Install 99 Packages 2026-02-22T21:43:39.462 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:43:39.462 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 95 M 2026-02-22T21:43:39.462 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 520 M 2026-02-22T21:43:39.463 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:43:39.660 INFO:teuthology.orchestra.run.trial104.stdout:(28/99): flexiblas-3.0.4-9.el9.x86_64.rpm 66 kB/s | 30 kB 00:00 2026-02-22T21:43:40.270 INFO:teuthology.orchestra.run.trial104.stdout:(29/99): ceph-mgr-diskprediction-local-19.2.3-6 3.6 MB/s | 7.4 MB 00:02 2026-02-22T21:43:40.688 INFO:teuthology.orchestra.run.trial104.stdout:(30/99): flexiblas-openblas-openmp-3.0.4-9.el9. 35 kB/s | 15 kB 00:00 2026-02-22T21:43:41.022 INFO:teuthology.orchestra.run.trial104.stdout:(31/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 737 kB/s | 246 kB 00:00 2026-02-22T21:43:41.057 INFO:teuthology.orchestra.run.trial048.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 47 kB/s | 6.5 kB 00:00 2026-02-22T21:43:41.122 INFO:teuthology.orchestra.run.trial104.stdout:(32/99): lua-5.4.4-4.el9.x86_64.rpm 1.8 MB/s | 188 kB 00:00 2026-02-22T21:43:41.220 INFO:teuthology.orchestra.run.trial104.stdout:(33/99): openblas-0.3.29-1.el9.x86_64.rpm 427 kB/s | 42 kB 00:00 2026-02-22T21:43:41.259 INFO:teuthology.orchestra.run.trial048.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 3.2 MB/s | 1.1 MB 00:00 2026-02-22T21:43:41.262 INFO:teuthology.orchestra.run.trial104.stdout:(34/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 3.9 MB/s | 17 MB 00:04 2026-02-22T21:43:41.319 INFO:teuthology.orchestra.run.trial104.stdout:(35/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 1.8 MB/s | 3.0 MB 00:01 2026-02-22T21:43:41.335 INFO:teuthology.orchestra.run.trial048.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 5.8 MB/s | 2.4 MB 00:00 2026-02-22T21:43:41.370 INFO:teuthology.orchestra.run.trial048.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 883 kB/s | 31 kB 00:00 2026-02-22T21:43:41.409 INFO:teuthology.orchestra.run.trial048.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 3.8 MB/s | 150 kB 00:00 2026-02-22T21:43:41.646 INFO:teuthology.orchestra.run.trial104.stdout:(36/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 12 MB/s | 5.3 MB 00:00 2026-02-22T21:43:41.713 INFO:teuthology.orchestra.run.trial048.stdout:(6/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 13 MB/s | 3.8 MB 00:00 2026-02-22T21:43:41.746 INFO:teuthology.orchestra.run.trial104.stdout:(37/99): python3-devel-3.9.25-3.el9.x86_64.rpm 2.4 MB/s | 244 kB 00:00 2026-02-22T21:43:41.820 INFO:teuthology.orchestra.run.trial104.stdout:(38/99): python3-babel-2.9.1-2.el9.noarch.rpm 12 MB/s | 6.0 MB 00:00 2026-02-22T21:43:41.846 INFO:teuthology.orchestra.run.trial104.stdout:(39/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 2.4 MB/s | 249 kB 00:00 2026-02-22T21:43:41.922 INFO:teuthology.orchestra.run.trial104.stdout:(40/99): python3-libstoragemgmt-1.10.1-1.el9.x8 1.7 MB/s | 177 kB 00:00 2026-02-22T21:43:41.951 INFO:teuthology.orchestra.run.trial104.stdout:(41/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 12 MB/s | 1.2 MB 00:00 2026-02-22T21:43:42.022 INFO:teuthology.orchestra.run.trial104.stdout:(42/99): python3-mako-1.1.4-6.el9.noarch.rpm 1.7 MB/s | 172 kB 00:00 2026-02-22T21:43:42.050 INFO:teuthology.orchestra.run.trial104.stdout:(43/99): python3-markupsafe-1.1.1-12.el9.x86_64 350 kB/s | 35 kB 00:00 2026-02-22T21:43:42.151 INFO:teuthology.orchestra.run.trial104.stdout:(44/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 4.3 MB/s | 442 kB 00:00 2026-02-22T21:43:42.251 INFO:teuthology.orchestra.run.trial104.stdout:(45/99): python3-packaging-20.9-5.el9.noarch.rp 775 kB/s | 77 kB 00:00 2026-02-22T21:43:42.252 INFO:teuthology.orchestra.run.trial048.stdout:(7/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 4.0 MB/s | 4.7 MB 00:01 2026-02-22T21:43:42.279 INFO:teuthology.orchestra.run.trial048.stdout:(8/99): ceph-mgr-diskprediction-local-19.2.3-68 13 MB/s | 7.4 MB 00:00 2026-02-22T21:43:42.294 INFO:teuthology.orchestra.run.trial048.stdout:(9/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 529 kB/s | 22 kB 00:00 2026-02-22T21:43:42.316 INFO:teuthology.orchestra.run.trial048.stdout:(10/99): ceph-mgr-modules-core-19.2.3-689.gf90f 6.7 MB/s | 253 kB 00:00 2026-02-22T21:43:42.329 INFO:teuthology.orchestra.run.trial048.stdout:(11/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 1.4 MB/s | 49 kB 00:00 2026-02-22T21:43:42.347 INFO:teuthology.orchestra.run.trial048.stdout:(12/99): ceph-prometheus-alerts-19.2.3-689.gf90 544 kB/s | 17 kB 00:00 2026-02-22T21:43:42.352 INFO:teuthology.orchestra.run.trial104.stdout:(46/99): python3-protobuf-3.14.0-17.el9.noarch. 2.6 MB/s | 267 kB 00:00 2026-02-22T21:43:42.406 INFO:teuthology.orchestra.run.trial048.stdout:(13/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 3.8 MB/s | 299 kB 00:00 2026-02-22T21:43:42.445 INFO:teuthology.orchestra.run.trial048.stdout:(14/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 14 MB/s | 17 MB 00:01 2026-02-22T21:43:42.446 INFO:teuthology.orchestra.run.trial048.stdout:(15/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 7.6 MB/s | 769 kB 00:00 2026-02-22T21:43:42.452 INFO:teuthology.orchestra.run.trial104.stdout:(47/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.5 MB/s | 159 kB 00:00 2026-02-22T21:43:42.523 INFO:teuthology.orchestra.run.trial104.stdout:(48/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 12 MB/s | 6.1 MB 00:00 2026-02-22T21:43:42.553 INFO:teuthology.orchestra.run.trial104.stdout:(49/99): python3-pyasn1-modules-0.4.8-6.el9.noa 2.7 MB/s | 279 kB 00:00 2026-02-22T21:43:42.622 INFO:teuthology.orchestra.run.trial104.stdout:(50/99): python3-requests-oauthlib-1.3.0-12.el9 538 kB/s | 54 kB 00:00 2026-02-22T21:43:42.722 INFO:teuthology.orchestra.run.trial104.stdout:(51/99): python3-toml-0.10.2-6.el9.noarch.rpm 421 kB/s | 42 kB 00:00 2026-02-22T21:43:42.723 INFO:teuthology.orchestra.run.trial048.stdout:(16/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 145 kB/s | 40 kB 00:00 2026-02-22T21:43:42.807 INFO:teuthology.orchestra.run.trial048.stdout:(17/99): libconfig-1.7.2-9.el9.x86_64.rpm 199 kB/s | 72 kB 00:00 2026-02-22T21:43:42.821 INFO:teuthology.orchestra.run.trial104.stdout:(52/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 1.9 MB/s | 189 kB 00:00 2026-02-22T21:43:42.858 INFO:teuthology.orchestra.run.trial104.stdout:(53/99): protobuf-3.14.0-17.el9.x86_64.rpm 644 kB/s | 1.0 MB 00:01 2026-02-22T21:43:42.889 INFO:teuthology.orchestra.run.trial104.stdout:(54/99): lua-devel-5.4.4-4.el9.x86_64.rpm 731 kB/s | 22 kB 00:00 2026-02-22T21:43:42.919 INFO:teuthology.orchestra.run.trial104.stdout:(55/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 918 kB/s | 90 kB 00:00 2026-02-22T21:43:42.933 INFO:teuthology.orchestra.run.trial104.stdout:(56/99): protobuf-compiler-3.14.0-17.el9.x86_64 19 MB/s | 862 kB 00:00 2026-02-22T21:43:42.949 INFO:teuthology.orchestra.run.trial048.stdout:(18/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 645 kB/s | 351 kB 00:00 2026-02-22T21:43:43.040 INFO:teuthology.orchestra.run.trial048.stdout:(19/99): libquadmath-11.5.0-14.el9.x86_64.rpm 791 kB/s | 184 kB 00:00 2026-02-22T21:43:43.067 INFO:teuthology.orchestra.run.trial048.stdout:(20/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.1 MB/s | 253 kB 00:00 2026-02-22T21:43:43.074 INFO:teuthology.orchestra.run.trial104.stdout:(57/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 37 MB/s | 19 MB 00:00 2026-02-22T21:43:43.087 INFO:teuthology.orchestra.run.trial104.stdout:(58/99): grpc-data-1.46.7-10.el9.noarch.rpm 126 kB/s | 19 kB 00:00 2026-02-22T21:43:43.162 INFO:teuthology.orchestra.run.trial048.stdout:(21/99): libgfortran-11.5.0-14.el9.x86_64.rpm 1.8 MB/s | 794 kB 00:00 2026-02-22T21:43:43.162 INFO:teuthology.orchestra.run.trial048.stdout:(22/99): python3-ply-3.11-14.el9.noarch.rpm 1.1 MB/s | 106 kB 00:00 2026-02-22T21:43:43.250 INFO:teuthology.orchestra.run.trial048.stdout:(23/99): python3-pyparsing-2.4.7-9.el9.noarch.r 1.7 MB/s | 150 kB 00:00 2026-02-22T21:43:43.260 INFO:teuthology.orchestra.run.trial048.stdout:(24/99): python3-pycparser-2.20-6.el9.noarch.rp 1.3 MB/s | 135 kB 00:00 2026-02-22T21:43:43.311 INFO:teuthology.orchestra.run.trial104.stdout:(59/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 1.4 MB/s | 551 kB 00:00 2026-02-22T21:43:43.336 INFO:teuthology.orchestra.run.trial048.stdout:(25/99): python3-requests-2.25.1-10.el9.noarch. 1.4 MB/s | 126 kB 00:00 2026-02-22T21:43:43.357 INFO:teuthology.orchestra.run.trial048.stdout:(26/99): python3-cryptography-36.0.1-5.el9.x86_ 3.9 MB/s | 1.2 MB 00:00 2026-02-22T21:43:43.364 INFO:teuthology.orchestra.run.trial104.stdout:(60/99): python3-asyncssh-2.13.2-5.el9.noarch.r 1.9 MB/s | 548 kB 00:00 2026-02-22T21:43:43.364 INFO:teuthology.orchestra.run.trial104.stdout:(61/99): python3-autocommand-2.2.2-8.el9.noarch 552 kB/s | 29 kB 00:00 2026-02-22T21:43:43.365 INFO:teuthology.orchestra.run.trial104.stdout:(62/99): luarocks-3.9.2-5.el9.noarch.rpm 520 kB/s | 151 kB 00:00 2026-02-22T21:43:43.372 INFO:teuthology.orchestra.run.trial048.stdout:(27/99): python3-urllib3-1.26.5-7.el9.noarch.rp 1.9 MB/s | 218 kB 00:00 2026-02-22T21:43:43.419 INFO:teuthology.orchestra.run.trial104.stdout:(63/99): python3-backports-tarfile-1.2.0-1.el9. 1.1 MB/s | 60 kB 00:00 2026-02-22T21:43:43.419 INFO:teuthology.orchestra.run.trial104.stdout:(64/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 796 kB/s | 43 kB 00:00 2026-02-22T21:43:43.424 INFO:teuthology.orchestra.run.trial104.stdout:(65/99): python3-cachetools-4.2.4-1.el9.noarch. 548 kB/s | 32 kB 00:00 2026-02-22T21:43:43.425 INFO:teuthology.orchestra.run.trial048.stdout:(28/99): unzip-6.0-59.el9.x86_64.rpm 2.0 MB/s | 182 kB 00:00 2026-02-22T21:43:43.446 INFO:teuthology.orchestra.run.trial048.stdout:(29/99): zip-3.0-35.el9.x86_64.rpm 2.9 MB/s | 266 kB 00:00 2026-02-22T21:43:43.471 INFO:teuthology.orchestra.run.trial104.stdout:(66/99): python3-certifi-2023.05.07-4.el9.noarc 270 kB/s | 14 kB 00:00 2026-02-22T21:43:43.481 INFO:teuthology.orchestra.run.trial104.stdout:(67/99): python3-cheroot-10.0.1-4.el9.noarch.rp 2.7 MB/s | 173 kB 00:00 2026-02-22T21:43:43.539 INFO:teuthology.orchestra.run.trial104.stdout:(68/99): python3-google-auth-2.45.0-1.el9.noarc 3.7 MB/s | 254 kB 00:00 2026-02-22T21:43:43.564 INFO:teuthology.orchestra.run.trial104.stdout:(69/99): python3-cherrypy-18.6.1-2.el9.noarch.r 2.5 MB/s | 358 kB 00:00 2026-02-22T21:43:43.600 INFO:teuthology.orchestra.run.trial104.stdout:(70/99): python3-grpcio-tools-1.46.7-10.el9.x86 2.3 MB/s | 144 kB 00:00 2026-02-22T21:43:43.627 INFO:teuthology.orchestra.run.trial104.stdout:(71/99): python3-influxdb-5.3.1-1.el9.noarch.rp 2.1 MB/s | 139 kB 00:00 2026-02-22T21:43:43.643 INFO:teuthology.orchestra.run.trial104.stdout:(72/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 13 MB/s | 2.0 MB 00:00 2026-02-22T21:43:43.655 INFO:teuthology.orchestra.run.trial104.stdout:(73/99): python3-isodate-0.6.1-3.el9.noarch.rpm 1.0 MB/s | 56 kB 00:00 2026-02-22T21:43:43.680 INFO:teuthology.orchestra.run.trial104.stdout:(74/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 205 kB/s | 11 kB 00:00 2026-02-22T21:43:43.695 INFO:teuthology.orchestra.run.trial104.stdout:(75/99): python3-jaraco-classes-3.2.1-5.el9.noa 337 kB/s | 18 kB 00:00 2026-02-22T21:43:43.707 INFO:teuthology.orchestra.run.trial104.stdout:(76/99): python3-jaraco-collections-3.0.0-8.el9 442 kB/s | 23 kB 00:00 2026-02-22T21:43:43.732 INFO:teuthology.orchestra.run.trial104.stdout:(77/99): python3-jaraco-context-6.0.1-3.el9.noa 376 kB/s | 20 kB 00:00 2026-02-22T21:43:43.748 INFO:teuthology.orchestra.run.trial104.stdout:(78/99): python3-jaraco-functools-3.5.0-2.el9.n 372 kB/s | 19 kB 00:00 2026-02-22T21:43:43.759 INFO:teuthology.orchestra.run.trial104.stdout:(79/99): python3-jaraco-text-4.0.0-2.el9.noarch 503 kB/s | 26 kB 00:00 2026-02-22T21:43:43.784 INFO:teuthology.orchestra.run.trial048.stdout:(30/99): flexiblas-3.0.4-9.el9.x86_64.rpm 72 kB/s | 30 kB 00:00 2026-02-22T21:43:43.801 INFO:teuthology.orchestra.run.trial104.stdout:(80/99): python3-logutils-0.3.5-21.el9.noarch.r 878 kB/s | 46 kB 00:00 2026-02-22T21:43:43.816 INFO:teuthology.orchestra.run.trial104.stdout:(81/99): python3-more-itertools-8.12.0-2.el9.no 1.4 MB/s | 79 kB 00:00 2026-02-22T21:43:43.845 INFO:teuthology.orchestra.run.trial048.stdout:(31/99): flexiblas-openblas-openmp-3.0.4-9.el9. 37 kB/s | 15 kB 00:00 2026-02-22T21:43:43.855 INFO:teuthology.orchestra.run.trial104.stdout:(82/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 1.6 MB/s | 86 kB 00:00 2026-02-22T21:43:43.871 INFO:teuthology.orchestra.run.trial104.stdout:(83/99): python3-natsort-7.1.1-5.el9.noarch.rpm 1.0 MB/s | 58 kB 00:00 2026-02-22T21:43:43.874 INFO:teuthology.orchestra.run.trial104.stdout:(84/99): python3-kubernetes-26.1.0-3.el9.noarch 7.2 MB/s | 1.0 MB 00:00 2026-02-22T21:43:43.911 INFO:teuthology.orchestra.run.trial104.stdout:(85/99): python3-pecan-1.4.2-3.el9.noarch.rpm 4.7 MB/s | 272 kB 00:00 2026-02-22T21:43:43.924 INFO:teuthology.orchestra.run.trial104.stdout:(86/99): python3-portend-3.1.0-2.el9.noarch.rpm 313 kB/s | 16 kB 00:00 2026-02-22T21:43:43.926 INFO:teuthology.orchestra.run.trial104.stdout:(87/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2026-02-22T21:43:43.965 INFO:teuthology.orchestra.run.trial104.stdout:(88/99): python3-repoze-lru-0.7-16.el9.noarch.r 571 kB/s | 31 kB 00:00 2026-02-22T21:43:43.978 INFO:teuthology.orchestra.run.trial104.stdout:(89/99): python3-rsa-4.9-2.el9.noarch.rpm 1.1 MB/s | 59 kB 00:00 2026-02-22T21:43:43.988 INFO:teuthology.orchestra.run.trial104.stdout:(90/99): python3-routes-2.5.1-5.el9.noarch.rpm 2.9 MB/s | 188 kB 00:00 2026-02-22T21:43:44.019 INFO:teuthology.orchestra.run.trial104.stdout:(91/99): python3-saml-1.16.0-1.el9.noarch.rpm 2.2 MB/s | 125 kB 00:00 2026-02-22T21:43:44.029 INFO:teuthology.orchestra.run.trial104.stdout:(92/99): python3-tempora-5.0.0-2.el9.noarch.rpm 695 kB/s | 36 kB 00:00 2026-02-22T21:43:44.044 INFO:teuthology.orchestra.run.trial104.stdout:(93/99): python3-typing-extensions-4.15.0-1.el9 1.5 MB/s | 86 kB 00:00 2026-02-22T21:43:44.074 INFO:teuthology.orchestra.run.trial104.stdout:(94/99): python3-webob-1.8.8-2.el9.noarch.rpm 4.1 MB/s | 230 kB 00:00 2026-02-22T21:43:44.082 INFO:teuthology.orchestra.run.trial104.stdout:(95/99): python3-websocket-client-1.2.3-2.el9.n 1.7 MB/s | 90 kB 00:00 2026-02-22T21:43:44.103 INFO:teuthology.orchestra.run.trial048.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 772 kB/s | 246 kB 00:00 2026-02-22T21:43:44.124 INFO:teuthology.orchestra.run.trial104.stdout:(96/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.3 MB/s | 427 kB 00:00 2026-02-22T21:43:44.127 INFO:teuthology.orchestra.run.trial104.stdout:(97/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 913 kB/s | 48 kB 00:00 2026-02-22T21:43:44.134 INFO:teuthology.orchestra.run.trial104.stdout:(98/99): python3-xmltodict-0.12.0-15.el9.noarch 429 kB/s | 22 kB 00:00 2026-02-22T21:43:44.161 INFO:teuthology.orchestra.run.trial048.stdout:(33/99): lua-5.4.4-4.el9.x86_64.rpm 595 kB/s | 188 kB 00:00 2026-02-22T21:43:44.176 INFO:teuthology.orchestra.run.trial104.stdout:(99/99): python3-zc-lockfile-2.0-10.el9.noarch. 380 kB/s | 20 kB 00:00 2026-02-22T21:43:44.181 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:44.181 INFO:teuthology.orchestra.run.trial104.stdout:Total 10 MB/s | 95 MB 00:09 2026-02-22T21:43:44.187 INFO:teuthology.orchestra.run.trial048.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 502 kB/s | 42 kB 00:00 2026-02-22T21:43:44.619 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:43:44.660 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:43:44.661 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:43:45.077 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:43:45.077 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:43:45.373 INFO:teuthology.orchestra.run.trial048.stdout:(35/99): protobuf-3.14.0-17.el9.x86_64.rpm 868 kB/s | 1.0 MB 00:01 2026-02-22T21:43:45.541 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:43:45.543 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T21:43:45.574 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T21:43:45.580 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T21:43:45.601 INFO:teuthology.orchestra.run.trial104.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T21:43:45.607 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T21:43:45.641 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T21:43:45.650 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T21:43:45.655 INFO:teuthology.orchestra.run.trial104.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T21:43:45.658 INFO:teuthology.orchestra.run.trial104.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T21:43:45.662 INFO:teuthology.orchestra.run.trial104.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T21:43:45.685 INFO:teuthology.orchestra.run.trial104.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T21:43:45.698 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T21:43:45.701 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T21:43:45.707 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T21:43:45.709 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T21:43:45.720 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T21:43:45.726 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T21:43:45.751 INFO:teuthology.orchestra.run.trial104.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T21:43:45.754 INFO:teuthology.orchestra.run.trial104.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T21:43:45.761 INFO:teuthology.orchestra.run.trial104.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T21:43:45.785 INFO:teuthology.orchestra.run.trial104.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T21:43:45.837 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T21:43:45.850 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T21:43:45.852 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T21:43:45.857 INFO:teuthology.orchestra.run.trial104.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T21:43:45.862 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T21:43:45.867 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T21:43:45.870 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T21:43:45.884 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T21:43:45.905 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T21:43:45.910 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T21:43:45.915 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T21:43:45.926 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T21:43:45.935 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T21:43:45.941 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T21:43:45.961 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T21:43:45.970 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T21:43:46.024 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T21:43:46.037 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T21:43:46.052 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T21:43:46.057 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T21:43:46.064 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T21:43:46.105 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T21:43:46.433 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T21:43:46.443 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T21:43:46.446 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T21:43:46.452 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T21:43:46.454 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T21:43:46.482 INFO:teuthology.orchestra.run.trial104.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T21:43:46.526 INFO:teuthology.orchestra.run.trial104.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T21:43:46.532 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T21:43:46.536 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T21:43:46.542 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T21:43:46.545 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T21:43:46.552 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T21:43:46.555 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T21:43:46.583 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T21:43:46.593 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T21:43:46.628 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T21:43:46.854 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T21:43:46.878 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T21:43:46.882 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T21:43:46.940 INFO:teuthology.orchestra.run.trial104.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T21:43:46.942 INFO:teuthology.orchestra.run.trial104.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T21:43:46.963 INFO:teuthology.orchestra.run.trial104.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T21:43:47.199 INFO:teuthology.orchestra.run.trial048.stdout:(36/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 811 kB/s | 3.0 MB 00:03 2026-02-22T21:43:47.289 INFO:teuthology.orchestra.run.trial104.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T21:43:47.363 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T21:43:47.519 INFO:teuthology.orchestra.run.trial048.stdout:(37/99): python3-devel-3.9.25-3.el9.x86_64.rpm 766 kB/s | 244 kB 00:00 2026-02-22T21:43:47.838 INFO:teuthology.orchestra.run.trial048.stdout:(38/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 778 kB/s | 249 kB 00:00 2026-02-22T21:43:48.035 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T21:43:48.051 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T21:43:48.061 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T21:43:48.068 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T21:43:48.079 INFO:teuthology.orchestra.run.trial048.stdout:(39/99): python3-libstoragemgmt-1.10.1-1.el9.x8 733 kB/s | 177 kB 00:00 2026-02-22T21:43:48.081 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T21:43:48.097 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T21:43:48.172 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T21:43:48.181 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T21:43:48.204 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T21:43:48.235 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T21:43:48.285 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T21:43:48.291 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T21:43:48.295 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T21:43:48.296 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T21:43:48.314 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:43:48.314 INFO:teuthology.orchestra.run.trial104.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T21:43:48.314 INFO:teuthology.orchestra.run.trial104.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T21:43:48.314 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:48.322 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:43:48.349 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:43:48.349 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T21:43:48.349 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:48.383 INFO:teuthology.orchestra.run.trial104.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T21:43:48.447 INFO:teuthology.orchestra.run.trial104.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T21:43:48.449 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:43:48.463 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:43:48.463 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:48.464 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T21:43:48.464 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:49.161 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:43:49.186 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:43:49.186 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:49.187 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T21:43:49.187 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:43:49.187 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:43:49.187 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:49.245 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:43:49.247 INFO:teuthology.orchestra.run.trial104.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:43:49.252 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T21:43:49.271 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T21:43:49.272 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:43:49.504 INFO:teuthology.orchestra.run.trial048.stdout:(40/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 879 kB/s | 1.2 MB 00:01 2026-02-22T21:43:49.746 INFO:teuthology.orchestra.run.trial048.stdout:(41/99): python3-mako-1.1.4-6.el9.noarch.rpm 710 kB/s | 172 kB 00:00 2026-02-22T21:43:49.763 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:43:49.767 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:43:49.831 INFO:teuthology.orchestra.run.trial048.stdout:(42/99): python3-markupsafe-1.1.1-12.el9.x86_64 412 kB/s | 35 kB 00:00 2026-02-22T21:43:50.179 INFO:teuthology.orchestra.run.trial048.stdout:(43/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 899 kB/s | 5.3 MB 00:06 2026-02-22T21:43:50.223 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:43:50.224 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:43:50.236 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:43:50.237 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:43:50.294 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:43:50.343 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T21:43:50.344 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:43:50.366 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:43:50.366 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:50.366 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T21:43:50.366 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:43:50.366 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:43:50.366 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:50.377 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:43:50.595 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:43:50.597 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:43:50.620 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:43:50.620 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:50.620 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T21:43:50.620 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:43:50.620 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:43:50.620 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:50.724 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:43:50.731 INFO:teuthology.orchestra.run.trial048.stdout:(44/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 801 kB/s | 442 kB 00:00 2026-02-22T21:43:50.745 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:43:50.746 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:50.746 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T21:43:50.746 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:43:50.746 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:43:50.746 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:50.764 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:43:50.895 INFO:teuthology.orchestra.run.trial048.stdout:(45/99): python3-packaging-20.9-5.el9.noarch.rp 469 kB/s | 77 kB 00:00 2026-02-22T21:43:51.217 INFO:teuthology.orchestra.run.trial048.stdout:(46/99): python3-protobuf-3.14.0-17.el9.noarch. 832 kB/s | 267 kB 00:00 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T21:43:51.408 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T21:43:51.409 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T21:43:51.410 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T21:43:51.457 INFO:teuthology.orchestra.run.trial048.stdout:(47/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 661 kB/s | 159 kB 00:00 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:51.476 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:43:51.477 INFO:teuthology.orchestra.run.trial104.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T21:43:51.478 INFO:teuthology.orchestra.run.trial104.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T21:43:51.479 INFO:teuthology.orchestra.run.trial104.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T21:43:51.480 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T21:43:51.481 INFO:teuthology.orchestra.run.trial104.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T21:43:51.482 INFO:teuthology.orchestra.run.trial104.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T21:43:51.483 INFO:teuthology.orchestra.run.trial104.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: zip-3.0-35.el9.x86_64 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:51.484 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:51.533 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-base 2026-02-22T21:43:51.778 INFO:teuthology.orchestra.run.trial048.stdout:(48/99): python3-pyasn1-modules-0.4.8-6.el9.noa 869 kB/s | 279 kB 00:00 2026-02-22T21:43:51.848 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:51.865 INFO:teuthology.orchestra.run.trial048.stdout:(49/99): python3-requests-oauthlib-1.3.0-12.el9 615 kB/s | 54 kB 00:00 2026-02-22T21:43:51.913 INFO:teuthology.orchestra.run.trial104.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:43:51.931 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:51.932 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:51.932 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:51.954 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install cephadm 2026-02-22T21:43:52.247 INFO:teuthology.orchestra.run.trial048.stdout:(50/99): python3-babel-2.9.1-2.el9.noarch.rpm 888 kB/s | 6.0 MB 00:06 2026-02-22T21:43:52.263 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:52.327 INFO:teuthology.orchestra.run.trial104.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:43:52.337 INFO:teuthology.orchestra.run.trial048.stdout:(51/99): python3-toml-0.10.2-6.el9.noarch.rpm 461 kB/s | 42 kB 00:00 2026-02-22T21:43:52.345 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:52.346 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:52.346 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:52.368 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T21:43:52.587 INFO:teuthology.orchestra.run.trial048.stdout:(52/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 755 kB/s | 189 kB 00:00 2026-02-22T21:43:52.678 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:52.749 INFO:teuthology.orchestra.run.trial048.stdout:(53/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 558 kB/s | 90 kB 00:00 2026-02-22T21:43:52.761 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout: Package Arch Version Repo Size 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:52.762 INFO:teuthology.orchestra.run.trial104.stdout:Install 1 Package 2026-02-22T21:43:52.763 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:52.763 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 145 k 2026-02-22T21:43:52.763 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 431 k 2026-02-22T21:43:52.763 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:43:52.779 INFO:teuthology.orchestra.run.trial048.stdout:(54/99): lua-devel-5.4.4-4.el9.x86_64.rpm 741 kB/s | 22 kB 00:00 2026-02-22T21:43:52.823 INFO:teuthology.orchestra.run.trial048.stdout:(55/99): protobuf-compiler-3.14.0-17.el9.x86_64 19 MB/s | 862 kB 00:00 2026-02-22T21:43:52.853 INFO:teuthology.orchestra.run.trial104.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.6 MB/s | 145 kB 00:00 2026-02-22T21:43:52.854 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:52.854 INFO:teuthology.orchestra.run.trial104.stdout:Total 1.5 MB/s | 145 kB 00:00 2026-02-22T21:43:52.854 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:43:52.857 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:43:52.857 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:43:52.888 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:43:52.889 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:43:52.944 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:43:52.945 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:43:52.969 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:43:52.969 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:43:52.969 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T21:43:52.969 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:53.144 INFO:teuthology.orchestra.run.trial048.stdout:(56/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 1.7 MB/s | 551 kB 00:00 2026-02-22T21:43:53.204 INFO:teuthology.orchestra.run.trial048.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 324 kB/s | 19 kB 00:00 2026-02-22T21:43:53.267 INFO:teuthology.orchestra.run.trial048.stdout:(58/99): luarocks-3.9.2-5.el9.noarch.rpm 2.4 MB/s | 151 kB 00:00 2026-02-22T21:43:53.276 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:43:53.276 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:53.276 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:43:53.276 INFO:teuthology.orchestra.run.trial104.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:53.276 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:53.276 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:53.304 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-mgr 2026-02-22T21:43:53.344 INFO:teuthology.orchestra.run.trial048.stdout:(59/99): python3-asyncssh-2.13.2-5.el9.noarch.r 6.9 MB/s | 548 kB 00:00 2026-02-22T21:43:53.392 INFO:teuthology.orchestra.run.trial048.stdout:(60/99): python3-autocommand-2.2.2-8.el9.noarch 616 kB/s | 29 kB 00:00 2026-02-22T21:43:53.439 INFO:teuthology.orchestra.run.trial048.stdout:(61/99): python3-backports-tarfile-1.2.0-1.el9. 1.3 MB/s | 60 kB 00:00 2026-02-22T21:43:53.488 INFO:teuthology.orchestra.run.trial048.stdout:(62/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 886 kB/s | 43 kB 00:00 2026-02-22T21:43:53.529 INFO:teuthology.orchestra.run.trial048.stdout:(63/99): python3-cachetools-4.2.4-1.el9.noarch. 773 kB/s | 32 kB 00:00 2026-02-22T21:43:53.578 INFO:teuthology.orchestra.run.trial048.stdout:(64/99): python3-certifi-2023.05.07-4.el9.noarc 291 kB/s | 14 kB 00:00 2026-02-22T21:43:53.612 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:53.626 INFO:teuthology.orchestra.run.trial048.stdout:(65/99): python3-cheroot-10.0.1-4.el9.noarch.rp 3.5 MB/s | 173 kB 00:00 2026-02-22T21:43:53.676 INFO:teuthology.orchestra.run.trial104.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:43:53.685 INFO:teuthology.orchestra.run.trial048.stdout:(66/99): python3-cherrypy-18.6.1-2.el9.noarch.r 6.0 MB/s | 358 kB 00:00 2026-02-22T21:43:53.694 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:53.695 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:53.695 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:53.715 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T21:43:53.736 INFO:teuthology.orchestra.run.trial048.stdout:(67/99): python3-google-auth-2.45.0-1.el9.noarc 4.9 MB/s | 254 kB 00:00 2026-02-22T21:43:53.856 INFO:teuthology.orchestra.run.trial048.stdout:(68/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 17 MB/s | 2.0 MB 00:00 2026-02-22T21:43:53.906 INFO:teuthology.orchestra.run.trial048.stdout:(69/99): python3-grpcio-tools-1.46.7-10.el9.x86 2.9 MB/s | 144 kB 00:00 2026-02-22T21:43:53.955 INFO:teuthology.orchestra.run.trial048.stdout:(70/99): python3-influxdb-5.3.1-1.el9.noarch.rp 2.8 MB/s | 139 kB 00:00 2026-02-22T21:43:54.010 INFO:teuthology.orchestra.run.trial048.stdout:(71/99): python3-isodate-0.6.1-3.el9.noarch.rpm 1.0 MB/s | 56 kB 00:00 2026-02-22T21:43:54.032 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:54.048 INFO:teuthology.orchestra.run.trial048.stdout:(72/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 283 kB/s | 11 kB 00:00 2026-02-22T21:43:54.086 INFO:teuthology.orchestra.run.trial048.stdout:(73/99): python3-jaraco-classes-3.2.1-5.el9.noa 465 kB/s | 18 kB 00:00 2026-02-22T21:43:54.096 INFO:teuthology.orchestra.run.trial104.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:43:54.114 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:54.115 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:54.115 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:54.124 INFO:teuthology.orchestra.run.trial048.stdout:(74/99): python3-jaraco-collections-3.0.0-8.el9 611 kB/s | 23 kB 00:00 2026-02-22T21:43:54.137 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T21:43:54.166 INFO:teuthology.orchestra.run.trial048.stdout:(75/99): python3-jaraco-context-6.0.1-3.el9.noa 466 kB/s | 20 kB 00:00 2026-02-22T21:43:54.205 INFO:teuthology.orchestra.run.trial048.stdout:(76/99): python3-jaraco-functools-3.5.0-2.el9.n 511 kB/s | 19 kB 00:00 2026-02-22T21:43:54.243 INFO:teuthology.orchestra.run.trial048.stdout:(77/99): python3-jaraco-text-4.0.0-2.el9.noarch 689 kB/s | 26 kB 00:00 2026-02-22T21:43:54.306 INFO:teuthology.orchestra.run.trial048.stdout:(78/99): python3-kubernetes-26.1.0-3.el9.noarch 16 MB/s | 1.0 MB 00:00 2026-02-22T21:43:54.354 INFO:teuthology.orchestra.run.trial048.stdout:(79/99): python3-logutils-0.3.5-21.el9.noarch.r 949 kB/s | 46 kB 00:00 2026-02-22T21:43:54.397 INFO:teuthology.orchestra.run.trial048.stdout:(80/99): python3-more-itertools-8.12.0-2.el9.no 1.8 MB/s | 79 kB 00:00 2026-02-22T21:43:54.436 INFO:teuthology.orchestra.run.trial048.stdout:(81/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 2.2 MB/s | 86 kB 00:00 2026-02-22T21:43:54.446 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:54.474 INFO:teuthology.orchestra.run.trial048.stdout:(82/99): python3-natsort-7.1.1-5.el9.noarch.rpm 1.5 MB/s | 58 kB 00:00 2026-02-22T21:43:54.511 INFO:teuthology.orchestra.run.trial104.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:43:54.524 INFO:teuthology.orchestra.run.trial048.stdout:(83/99): python3-pecan-1.4.2-3.el9.noarch.rpm 5.3 MB/s | 272 kB 00:00 2026-02-22T21:43:54.529 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:54.529 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:54.530 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:54.551 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-mgr-rook 2026-02-22T21:43:54.573 INFO:teuthology.orchestra.run.trial048.stdout:(84/99): python3-portend-3.1.0-2.el9.noarch.rpm 336 kB/s | 16 kB 00:00 2026-02-22T21:43:54.613 INFO:teuthology.orchestra.run.trial048.stdout:(85/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 2.3 MB/s | 90 kB 00:00 2026-02-22T21:43:54.661 INFO:teuthology.orchestra.run.trial048.stdout:(86/99): python3-repoze-lru-0.7-16.el9.noarch.r 634 kB/s | 31 kB 00:00 2026-02-22T21:43:54.710 INFO:teuthology.orchestra.run.trial048.stdout:(87/99): python3-routes-2.5.1-5.el9.noarch.rpm 3.8 MB/s | 188 kB 00:00 2026-02-22T21:43:54.758 INFO:teuthology.orchestra.run.trial048.stdout:(88/99): python3-rsa-4.9-2.el9.noarch.rpm 1.2 MB/s | 59 kB 00:00 2026-02-22T21:43:54.801 INFO:teuthology.orchestra.run.trial048.stdout:(89/99): python3-saml-1.16.0-1.el9.noarch.rpm 2.9 MB/s | 125 kB 00:00 2026-02-22T21:43:54.850 INFO:teuthology.orchestra.run.trial048.stdout:(90/99): python3-tempora-5.0.0-2.el9.noarch.rpm 734 kB/s | 36 kB 00:00 2026-02-22T21:43:54.858 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:54.892 INFO:teuthology.orchestra.run.trial048.stdout:(91/99): python3-typing-extensions-4.15.0-1.el9 2.0 MB/s | 86 kB 00:00 2026-02-22T21:43:54.927 INFO:teuthology.orchestra.run.trial104.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:43:54.942 INFO:teuthology.orchestra.run.trial048.stdout:(92/99): python3-webob-1.8.8-2.el9.noarch.rpm 4.6 MB/s | 230 kB 00:00 2026-02-22T21:43:54.945 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:54.946 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:54.946 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:54.966 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T21:43:54.988 INFO:teuthology.orchestra.run.trial048.stdout:(93/99): python3-websocket-client-1.2.3-2.el9.n 1.9 MB/s | 90 kB 00:00 2026-02-22T21:43:55.031 INFO:teuthology.orchestra.run.trial048.stdout:(94/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 9.7 MB/s | 427 kB 00:00 2026-02-22T21:43:55.080 INFO:teuthology.orchestra.run.trial048.stdout:(95/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 990 kB/s | 48 kB 00:00 2026-02-22T21:43:55.122 INFO:teuthology.orchestra.run.trial048.stdout:(96/99): python3-xmltodict-0.12.0-15.el9.noarch 523 kB/s | 22 kB 00:00 2026-02-22T21:43:55.160 INFO:teuthology.orchestra.run.trial048.stdout:(97/99): python3-zc-lockfile-2.0-10.el9.noarch. 524 kB/s | 20 kB 00:00 2026-02-22T21:43:55.276 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:55.341 INFO:teuthology.orchestra.run.trial104.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:43:55.359 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:55.360 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:55.360 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:55.380 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-fuse 2026-02-22T21:43:55.687 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:55.770 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:55.771 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:55.771 INFO:teuthology.orchestra.run.trial104.stdout: Package Architecture Version Repository Size 2026-02-22T21:43:55.771 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:55.771 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:43:55.771 INFO:teuthology.orchestra.run.trial104.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T21:43:55.771 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:55.772 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:43:55.772 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:55.772 INFO:teuthology.orchestra.run.trial104.stdout:Install 1 Package 2026-02-22T21:43:55.772 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:55.772 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 1.2 M 2026-02-22T21:43:55.772 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 3.6 M 2026-02-22T21:43:55.772 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:43:56.590 INFO:teuthology.orchestra.run.trial104.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.4 MB/s | 1.2 MB 00:00 2026-02-22T21:43:56.590 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:56.590 INFO:teuthology.orchestra.run.trial104.stdout:Total 1.4 MB/s | 1.2 MB 00:00 2026-02-22T21:43:56.590 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:43:56.594 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:43:56.594 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:43:56.631 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:43:56.631 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:43:56.737 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:43:56.754 INFO:teuthology.orchestra.run.trial104.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:56.898 INFO:teuthology.orchestra.run.trial048.stdout:(98/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 888 kB/s | 6.1 MB 00:07 2026-02-22T21:43:56.986 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:57.043 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:57.043 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:57.043 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:43:57.043 INFO:teuthology.orchestra.run.trial104.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:57.043 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:57.043 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:57.071 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install ceph-volume 2026-02-22T21:43:57.386 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:57.451 INFO:teuthology.orchestra.run.trial104.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:43:57.469 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:57.469 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:57.470 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:57.491 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install librados-devel 2026-02-22T21:43:57.797 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout: Package Arch Version Repo Size 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:Install 1 Package 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 126 k 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 456 k 2026-02-22T21:43:57.880 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:43:57.967 INFO:teuthology.orchestra.run.trial104.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T21:43:57.967 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:57.967 INFO:teuthology.orchestra.run.trial104.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T21:43:57.967 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:43:57.970 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:43:57.970 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:43:58.003 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:43:58.004 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:43:58.059 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:43:58.075 INFO:teuthology.orchestra.run.trial104.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:58.155 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:58.214 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:58.214 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:58.214 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:43:58.214 INFO:teuthology.orchestra.run.trial104.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:58.214 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:58.214 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:58.242 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install libcephfs2 2026-02-22T21:43:58.558 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:58.623 INFO:teuthology.orchestra.run.trial104.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:43:58.641 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:58.642 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:58.642 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:58.664 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install libcephfs-devel 2026-02-22T21:43:58.973 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:59.056 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout: Package Arch Version Repo Size 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout:Install 1 Package 2026-02-22T21:43:59.057 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:59.058 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 34 k 2026-02-22T21:43:59.058 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 153 k 2026-02-22T21:43:59.058 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:43:59.120 INFO:teuthology.orchestra.run.trial104.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 536 kB/s | 34 kB 00:00 2026-02-22T21:43:59.120 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:43:59.121 INFO:teuthology.orchestra.run.trial104.stdout:Total 529 kB/s | 34 kB 00:00 2026-02-22T21:43:59.121 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:43:59.122 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:43:59.122 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:43:59.125 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:43:59.125 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:43:59.146 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:43:59.162 INFO:teuthology.orchestra.run.trial104.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:59.216 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:59.273 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:43:59.273 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:59.273 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:43:59.273 INFO:teuthology.orchestra.run.trial104.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:43:59.273 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:43:59.273 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:59.300 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install librados2 2026-02-22T21:43:59.610 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:43:59.674 INFO:teuthology.orchestra.run.trial104.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:43:59.692 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:43:59.693 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:43:59.693 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:43:59.715 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install librbd1 2026-02-22T21:44:00.024 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:00.089 INFO:teuthology.orchestra.run.trial104.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:00.107 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:00.108 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:44:00.108 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:00.130 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install python3-rados 2026-02-22T21:44:00.439 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:00.504 INFO:teuthology.orchestra.run.trial104.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:00.523 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:00.523 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:44:00.523 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:00.545 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install python3-rgw 2026-02-22T21:44:00.855 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:00.920 INFO:teuthology.orchestra.run.trial104.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:00.938 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:00.939 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:44:00.939 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:00.961 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install python3-cephfs 2026-02-22T21:44:01.272 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:45 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:01.336 INFO:teuthology.orchestra.run.trial104.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:01.355 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:01.355 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:44:01.356 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:01.376 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install python3-rbd 2026-02-22T21:44:01.690 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:45 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:01.756 INFO:teuthology.orchestra.run.trial104.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:01.774 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:01.775 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:44:01.775 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:01.798 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install rbd-fuse 2026-02-22T21:44:02.110 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:46 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:02.192 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout: Package Architecture Version Repository Size 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:Install 1 Package 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 85 k 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 227 k 2026-02-22T21:44:02.193 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:44:02.522 INFO:teuthology.orchestra.run.trial104.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 260 kB/s | 85 kB 00:00 2026-02-22T21:44:02.522 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:02.523 INFO:teuthology.orchestra.run.trial104.stdout:Total 259 kB/s | 85 kB 00:00 2026-02-22T21:44:02.523 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:44:02.526 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:44:02.526 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:44:02.557 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:44:02.558 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:44:02.608 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:44:02.624 INFO:teuthology.orchestra.run.trial104.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:02.705 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:02.762 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:02.763 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:02.763 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:44:02.763 INFO:teuthology.orchestra.run.trial104.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:02.763 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:02.763 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:02.790 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install rbd-mirror 2026-02-22T21:44:03.106 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:47 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:03.189 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:03.190 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:03.190 INFO:teuthology.orchestra.run.trial104.stdout: Package Arch Version Repo Size 2026-02-22T21:44:03.190 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout:Install 1 Package 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 3.1 M 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 13 M 2026-02-22T21:44:03.191 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:44:03.581 INFO:teuthology.orchestra.run.trial104.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 8.0 MB/s | 3.1 MB 00:00 2026-02-22T21:44:03.581 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:03.581 INFO:teuthology.orchestra.run.trial104.stdout:Total 8.0 MB/s | 3.1 MB 00:00 2026-02-22T21:44:03.581 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:44:03.585 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:44:03.585 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:44:03.625 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:44:03.626 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:44:03.818 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:44:03.819 INFO:teuthology.orchestra.run.trial104.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:03.843 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:03.844 INFO:teuthology.orchestra.run.trial104.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:03.844 INFO:teuthology.orchestra.run.trial104.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T21:44:03.844 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:44:03.844 INFO:teuthology.orchestra.run.trial104.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:44:03.844 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:04.160 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:04.160 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:04.160 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:44:04.161 INFO:teuthology.orchestra.run.trial104.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:04.161 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:04.161 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:04.189 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install rbd-nbd 2026-02-22T21:44:04.503 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:04.587 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:04.587 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:04.587 INFO:teuthology.orchestra.run.trial104.stdout: Package Architecture Version Repository Size 2026-02-22T21:44:04.587 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:04.587 INFO:teuthology.orchestra.run.trial104.stdout:Installing: 2026-02-22T21:44:04.587 INFO:teuthology.orchestra.run.trial104.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T21:44:04.587 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:04.588 INFO:teuthology.orchestra.run.trial104.stdout:Transaction Summary 2026-02-22T21:44:04.588 INFO:teuthology.orchestra.run.trial104.stdout:================================================================================ 2026-02-22T21:44:04.588 INFO:teuthology.orchestra.run.trial104.stdout:Install 1 Package 2026-02-22T21:44:04.588 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:04.588 INFO:teuthology.orchestra.run.trial104.stdout:Total download size: 170 k 2026-02-22T21:44:04.588 INFO:teuthology.orchestra.run.trial104.stdout:Installed size: 490 k 2026-02-22T21:44:04.588 INFO:teuthology.orchestra.run.trial104.stdout:Downloading Packages: 2026-02-22T21:44:04.965 INFO:teuthology.orchestra.run.trial104.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 452 kB/s | 170 kB 00:00 2026-02-22T21:44:04.965 INFO:teuthology.orchestra.run.trial104.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:04.965 INFO:teuthology.orchestra.run.trial104.stdout:Total 451 kB/s | 170 kB 00:00 2026-02-22T21:44:04.965 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction check 2026-02-22T21:44:04.969 INFO:teuthology.orchestra.run.trial104.stdout:Transaction check succeeded. 2026-02-22T21:44:04.969 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction test 2026-02-22T21:44:05.000 INFO:teuthology.orchestra.run.trial104.stdout:Transaction test succeeded. 2026-02-22T21:44:05.000 INFO:teuthology.orchestra.run.trial104.stdout:Running transaction 2026-02-22T21:44:05.056 INFO:teuthology.orchestra.run.trial104.stdout: Preparing : 1/1 2026-02-22T21:44:05.073 INFO:teuthology.orchestra.run.trial104.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:05.161 INFO:teuthology.orchestra.run.trial104.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:05.219 INFO:teuthology.orchestra.run.trial104.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:05.219 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:05.219 INFO:teuthology.orchestra.run.trial104.stdout:Installed: 2026-02-22T21:44:05.220 INFO:teuthology.orchestra.run.trial104.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:05.220 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:05.220 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:05.245 DEBUG:teuthology.orchestra.run.trial104:> sudo yum -y install rbd-mirror 2026-02-22T21:44:05.552 INFO:teuthology.orchestra.run.trial104.stdout:Last metadata expiration check: 0:00:49 ago on Sun 22 Feb 2026 09:43:16 PM UTC. 2026-02-22T21:44:05.618 INFO:teuthology.orchestra.run.trial104.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:05.636 INFO:teuthology.orchestra.run.trial104.stdout:Dependencies resolved. 2026-02-22T21:44:05.636 INFO:teuthology.orchestra.run.trial104.stdout:Nothing to do. 2026-02-22T21:44:05.636 INFO:teuthology.orchestra.run.trial104.stdout:Complete! 2026-02-22T21:44:05.659 DEBUG:teuthology.parallel:result is None 2026-02-22T21:44:07.321 INFO:teuthology.orchestra.run.trial048.stdout:(99/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 1.2 MB/s | 19 MB 00:15 2026-02-22T21:44:07.324 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:07.324 INFO:teuthology.orchestra.run.trial048.stdout:Total 3.4 MB/s | 95 MB 00:27 2026-02-22T21:44:07.731 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:07.769 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:07.770 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:08.190 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:08.191 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:08.663 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:08.665 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T21:44:08.696 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T21:44:08.702 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T21:44:08.723 INFO:teuthology.orchestra.run.trial048.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T21:44:08.729 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T21:44:08.762 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T21:44:08.772 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T21:44:08.777 INFO:teuthology.orchestra.run.trial048.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T21:44:08.779 INFO:teuthology.orchestra.run.trial048.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T21:44:08.784 INFO:teuthology.orchestra.run.trial048.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T21:44:08.807 INFO:teuthology.orchestra.run.trial048.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T21:44:08.820 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T21:44:08.823 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T21:44:08.830 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T21:44:08.831 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T21:44:08.843 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T21:44:08.849 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T21:44:08.873 INFO:teuthology.orchestra.run.trial048.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T21:44:08.876 INFO:teuthology.orchestra.run.trial048.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T21:44:08.883 INFO:teuthology.orchestra.run.trial048.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T21:44:08.908 INFO:teuthology.orchestra.run.trial048.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T21:44:08.960 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T21:44:08.973 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T21:44:08.975 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T21:44:08.980 INFO:teuthology.orchestra.run.trial048.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T21:44:08.985 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T21:44:08.990 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T21:44:08.993 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T21:44:09.007 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T21:44:09.028 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T21:44:09.033 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T21:44:09.038 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T21:44:09.049 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T21:44:09.058 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T21:44:09.064 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T21:44:09.084 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T21:44:09.093 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T21:44:09.148 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T21:44:09.161 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T21:44:09.176 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T21:44:09.181 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T21:44:09.189 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T21:44:09.229 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T21:44:09.557 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T21:44:09.567 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T21:44:09.571 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T21:44:09.576 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T21:44:09.578 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T21:44:09.606 INFO:teuthology.orchestra.run.trial048.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T21:44:09.652 INFO:teuthology.orchestra.run.trial048.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T21:44:09.658 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T21:44:09.662 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T21:44:09.668 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T21:44:09.671 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T21:44:09.678 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T21:44:09.682 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T21:44:09.710 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T21:44:09.720 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T21:44:09.755 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T21:44:09.982 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T21:44:10.006 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T21:44:10.009 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T21:44:10.068 INFO:teuthology.orchestra.run.trial048.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T21:44:10.070 INFO:teuthology.orchestra.run.trial048.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T21:44:10.092 INFO:teuthology.orchestra.run.trial048.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T21:44:10.421 INFO:teuthology.orchestra.run.trial048.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T21:44:10.494 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T21:44:11.168 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T21:44:11.185 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T21:44:11.195 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T21:44:11.201 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T21:44:11.215 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T21:44:11.231 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T21:44:11.306 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T21:44:11.315 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T21:44:11.338 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T21:44:11.368 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T21:44:11.419 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T21:44:11.425 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T21:44:11.429 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T21:44:11.430 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T21:44:11.448 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:44:11.448 INFO:teuthology.orchestra.run.trial048.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T21:44:11.449 INFO:teuthology.orchestra.run.trial048.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T21:44:11.449 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:11.456 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:44:11.481 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T21:44:11.481 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T21:44:11.481 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:11.517 INFO:teuthology.orchestra.run.trial048.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T21:44:11.582 INFO:teuthology.orchestra.run.trial048.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T21:44:11.585 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:44:11.598 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T21:44:11.598 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:11.599 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T21:44:11.599 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:12.300 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:44:12.324 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T21:44:12.324 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:12.324 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T21:44:12.324 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:44:12.324 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T21:44:12.324 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:12.385 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:44:12.387 INFO:teuthology.orchestra.run.trial048.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T21:44:12.392 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T21:44:12.412 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T21:44:12.413 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:44:12.905 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T21:44:12.908 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:44:13.362 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T21:44:13.363 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:44:13.377 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T21:44:13.377 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:44:13.433 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T21:44:13.483 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T21:44:13.484 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:44:13.505 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T21:44:13.505 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:13.505 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T21:44:13.505 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:44:13.505 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T21:44:13.505 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:13.517 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:44:13.734 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T21:44:13.735 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:44:13.756 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T21:44:13.756 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:13.756 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T21:44:13.756 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:44:13.756 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T21:44:13.756 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:13.858 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:44:13.879 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T21:44:13.879 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:13.879 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T21:44:13.879 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:44:13.880 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T21:44:13.880 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:13.897 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T21:44:14.571 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T21:44:14.572 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T21:44:14.573 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T21:44:14.574 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T21:44:14.575 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T21:44:14.576 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T21:44:14.577 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T21:44:14.578 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T21:44:14.579 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T21:44:14.579 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T21:44:14.579 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T21:44:14.633 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T21:44:14.633 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:14.633 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.634 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T21:44:14.635 INFO:teuthology.orchestra.run.trial048.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T21:44:14.636 INFO:teuthology.orchestra.run.trial048.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T21:44:14.637 INFO:teuthology.orchestra.run.trial048.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T21:44:14.638 INFO:teuthology.orchestra.run.trial048.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T21:44:14.639 INFO:teuthology.orchestra.run.trial048.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T21:44:14.640 INFO:teuthology.orchestra.run.trial048.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T21:44:14.641 INFO:teuthology.orchestra.run.trial048.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T21:44:14.642 INFO:teuthology.orchestra.run.trial048.stdout: zip-3.0-35.el9.x86_64 2026-02-22T21:44:14.642 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:14.642 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:14.691 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-base 2026-02-22T21:44:15.006 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:57 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:15.072 INFO:teuthology.orchestra.run.trial048.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:15.090 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:15.090 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:15.090 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:15.111 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install cephadm 2026-02-22T21:44:15.419 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:57 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:15.484 INFO:teuthology.orchestra.run.trial048.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:44:15.502 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:15.535 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:15.536 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:15.536 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T21:44:15.849 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:57 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:15.933 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout: Package Arch Version Repo Size 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:44:15.934 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:15.935 INFO:teuthology.orchestra.run.trial048.stdout:Install 1 Package 2026-02-22T21:44:15.935 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:15.935 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 145 k 2026-02-22T21:44:15.935 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 431 k 2026-02-22T21:44:15.935 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:44:16.023 INFO:teuthology.orchestra.run.trial048.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.6 MB/s | 145 kB 00:00 2026-02-22T21:44:16.023 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:16.023 INFO:teuthology.orchestra.run.trial048.stdout:Total 1.6 MB/s | 145 kB 00:00 2026-02-22T21:44:16.023 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:16.026 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:16.026 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:16.058 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:16.058 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:16.113 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:16.114 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:44:16.138 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:44:16.138 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:16.138 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T21:44:16.139 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:16.459 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T21:44:16.459 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:16.459 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:16.459 INFO:teuthology.orchestra.run.trial048.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:16.459 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:16.459 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:16.485 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-mgr 2026-02-22T21:44:16.794 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:58 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:16.860 INFO:teuthology.orchestra.run.trial048.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:16.878 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:16.879 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:16.879 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:16.900 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T21:44:17.214 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:59 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:17.279 INFO:teuthology.orchestra.run.trial048.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:44:17.298 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:17.298 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:17.299 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:17.319 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T21:44:17.630 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:00:59 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:17.696 INFO:teuthology.orchestra.run.trial048.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:44:17.715 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:17.716 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:17.716 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:17.737 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-mgr-rook 2026-02-22T21:44:18.049 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:00 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:18.114 INFO:teuthology.orchestra.run.trial048.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:44:18.132 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:18.132 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:18.132 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:18.154 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T21:44:18.463 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:00 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:18.529 INFO:teuthology.orchestra.run.trial048.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:44:18.547 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:18.548 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:18.548 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:18.569 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-fuse 2026-02-22T21:44:18.877 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:00 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:18.961 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout: Package Architecture Version Repository Size 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:18.962 INFO:teuthology.orchestra.run.trial048.stdout:Install 1 Package 2026-02-22T21:44:18.963 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:18.963 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 1.2 M 2026-02-22T21:44:18.963 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 3.6 M 2026-02-22T21:44:18.963 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:44:19.106 INFO:teuthology.orchestra.run.trial048.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 8.0 MB/s | 1.2 MB 00:00 2026-02-22T21:44:19.106 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:19.106 INFO:teuthology.orchestra.run.trial048.stdout:Total 8.0 MB/s | 1.2 MB 00:00 2026-02-22T21:44:19.106 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:19.110 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:19.110 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:19.146 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:19.146 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:19.248 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:19.265 INFO:teuthology.orchestra.run.trial048.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:19.528 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:19.584 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:19.584 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:19.585 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:19.585 INFO:teuthology.orchestra.run.trial048.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:19.585 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:19.585 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:19.612 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install ceph-volume 2026-02-22T21:44:19.927 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:01 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:19.992 INFO:teuthology.orchestra.run.trial048.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T21:44:20.011 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:20.012 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:20.012 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:20.034 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install librados-devel 2026-02-22T21:44:20.341 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:02 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:20.424 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout: Package Arch Version Repo Size 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:Install 1 Package 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 126 k 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 456 k 2026-02-22T21:44:20.425 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:44:20.513 INFO:teuthology.orchestra.run.trial048.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T21:44:20.513 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:20.513 INFO:teuthology.orchestra.run.trial048.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T21:44:20.513 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:20.515 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:20.515 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:20.548 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:20.548 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:20.603 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:20.620 INFO:teuthology.orchestra.run.trial048.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:20.705 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:20.764 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:20.764 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:20.765 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:20.765 INFO:teuthology.orchestra.run.trial048.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:20.765 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:20.765 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:20.792 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install libcephfs2 2026-02-22T21:44:21.100 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:03 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:21.165 INFO:teuthology.orchestra.run.trial048.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:21.184 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:21.184 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:21.184 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:21.205 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install libcephfs-devel 2026-02-22T21:44:21.515 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:03 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:21.599 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:21.599 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:21.599 INFO:teuthology.orchestra.run.trial048.stdout: Package Arch Version Repo Size 2026-02-22T21:44:21.599 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:21.599 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:44:21.599 INFO:teuthology.orchestra.run.trial048.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout:Install 1 Package 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 34 k 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 153 k 2026-02-22T21:44:21.600 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:44:21.663 INFO:teuthology.orchestra.run.trial048.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 525 kB/s | 34 kB 00:00 2026-02-22T21:44:21.664 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:21.664 INFO:teuthology.orchestra.run.trial048.stdout:Total 517 kB/s | 34 kB 00:00 2026-02-22T21:44:21.664 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:21.666 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:21.666 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:21.669 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:21.669 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:21.691 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:21.707 INFO:teuthology.orchestra.run.trial048.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:21.761 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:21.817 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:21.817 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:21.817 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:21.817 INFO:teuthology.orchestra.run.trial048.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:21.817 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:21.817 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:21.844 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install librados2 2026-02-22T21:44:22.157 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:04 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:22.223 INFO:teuthology.orchestra.run.trial048.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:22.241 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:22.242 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:22.242 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:22.264 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install librbd1 2026-02-22T21:44:22.580 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:04 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:22.645 INFO:teuthology.orchestra.run.trial048.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:22.664 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:22.664 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:22.665 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:22.686 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install python3-rados 2026-02-22T21:44:23.002 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:05 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:23.068 INFO:teuthology.orchestra.run.trial048.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:23.086 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:23.086 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:23.086 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:23.108 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install python3-rgw 2026-02-22T21:44:23.414 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:05 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:23.478 INFO:teuthology.orchestra.run.trial048.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:23.496 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:23.497 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:23.497 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:23.518 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install python3-cephfs 2026-02-22T21:44:23.826 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:05 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:23.890 INFO:teuthology.orchestra.run.trial048.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:23.909 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:23.909 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:23.909 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:23.930 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install python3-rbd 2026-02-22T21:44:24.237 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:06 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:24.302 INFO:teuthology.orchestra.run.trial048.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:24.320 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:24.321 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:24.322 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:24.343 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install rbd-fuse 2026-02-22T21:44:24.654 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:06 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:24.739 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:24.739 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:24.739 INFO:teuthology.orchestra.run.trial048.stdout: Package Architecture Version Repository Size 2026-02-22T21:44:24.739 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:24.739 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:44:24.739 INFO:teuthology.orchestra.run.trial048.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T21:44:24.739 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:24.740 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:44:24.740 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:24.740 INFO:teuthology.orchestra.run.trial048.stdout:Install 1 Package 2026-02-22T21:44:24.740 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:24.740 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 85 k 2026-02-22T21:44:24.740 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 227 k 2026-02-22T21:44:24.740 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:44:24.816 INFO:teuthology.orchestra.run.trial048.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.1 MB/s | 85 kB 00:00 2026-02-22T21:44:24.816 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:24.816 INFO:teuthology.orchestra.run.trial048.stdout:Total 1.1 MB/s | 85 kB 00:00 2026-02-22T21:44:24.816 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:24.819 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:24.819 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:24.850 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:24.850 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:24.901 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:24.918 INFO:teuthology.orchestra.run.trial048.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:25.001 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:25.058 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:25.059 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:25.059 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:25.059 INFO:teuthology.orchestra.run.trial048.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:25.059 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:25.059 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:25.087 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install rbd-mirror 2026-02-22T21:44:25.400 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:07 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:25.484 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:25.484 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:25.484 INFO:teuthology.orchestra.run.trial048.stdout: Package Arch Version Repo Size 2026-02-22T21:44:25.484 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:25.484 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:44:25.484 INFO:teuthology.orchestra.run.trial048.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout:Install 1 Package 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 3.1 M 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 13 M 2026-02-22T21:44:25.485 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:44:25.640 INFO:teuthology.orchestra.run.trial048.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 20 MB/s | 3.1 MB 00:00 2026-02-22T21:44:25.640 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:25.640 INFO:teuthology.orchestra.run.trial048.stdout:Total 20 MB/s | 3.1 MB 00:00 2026-02-22T21:44:25.641 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:25.644 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:25.644 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:25.685 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:25.685 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:25.876 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:25.877 INFO:teuthology.orchestra.run.trial048.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:25.901 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:25.901 INFO:teuthology.orchestra.run.trial048.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T21:44:25.901 INFO:teuthology.orchestra.run.trial048.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T21:44:25.901 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:44:25.902 INFO:teuthology.orchestra.run.trial048.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T21:44:25.902 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:26.201 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:26.201 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:26.202 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:26.202 INFO:teuthology.orchestra.run.trial048.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:26.202 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:26.202 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:26.230 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install rbd-nbd 2026-02-22T21:44:26.543 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:08 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:26.628 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:26.628 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:26.628 INFO:teuthology.orchestra.run.trial048.stdout: Package Architecture Version Repository Size 2026-02-22T21:44:26.628 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:26.628 INFO:teuthology.orchestra.run.trial048.stdout:Installing: 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout:Transaction Summary 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout:================================================================================ 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout:Install 1 Package 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout:Total download size: 170 k 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout:Installed size: 490 k 2026-02-22T21:44:26.629 INFO:teuthology.orchestra.run.trial048.stdout:Downloading Packages: 2026-02-22T21:44:26.983 INFO:teuthology.orchestra.run.trial048.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 482 kB/s | 170 kB 00:00 2026-02-22T21:44:26.983 INFO:teuthology.orchestra.run.trial048.stdout:-------------------------------------------------------------------------------- 2026-02-22T21:44:26.983 INFO:teuthology.orchestra.run.trial048.stdout:Total 480 kB/s | 170 kB 00:00 2026-02-22T21:44:26.983 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction check 2026-02-22T21:44:26.987 INFO:teuthology.orchestra.run.trial048.stdout:Transaction check succeeded. 2026-02-22T21:44:26.987 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction test 2026-02-22T21:44:27.019 INFO:teuthology.orchestra.run.trial048.stdout:Transaction test succeeded. 2026-02-22T21:44:27.019 INFO:teuthology.orchestra.run.trial048.stdout:Running transaction 2026-02-22T21:44:27.074 INFO:teuthology.orchestra.run.trial048.stdout: Preparing : 1/1 2026-02-22T21:44:27.090 INFO:teuthology.orchestra.run.trial048.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:27.178 INFO:teuthology.orchestra.run.trial048.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:27.236 INFO:teuthology.orchestra.run.trial048.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T21:44:27.236 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:27.236 INFO:teuthology.orchestra.run.trial048.stdout:Installed: 2026-02-22T21:44:27.236 INFO:teuthology.orchestra.run.trial048.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T21:44:27.236 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:27.236 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:27.264 DEBUG:teuthology.orchestra.run.trial048:> sudo yum -y install rbd-mirror 2026-02-22T21:44:27.581 INFO:teuthology.orchestra.run.trial048.stdout:Last metadata expiration check: 0:01:09 ago on Sun 22 Feb 2026 09:43:18 PM UTC. 2026-02-22T21:44:27.645 INFO:teuthology.orchestra.run.trial048.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T21:44:27.663 INFO:teuthology.orchestra.run.trial048.stdout:Dependencies resolved. 2026-02-22T21:44:27.664 INFO:teuthology.orchestra.run.trial048.stdout:Nothing to do. 2026-02-22T21:44:27.664 INFO:teuthology.orchestra.run.trial048.stdout:Complete! 2026-02-22T21:44:27.688 DEBUG:teuthology.parallel:result is None 2026-02-22T21:44:27.689 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:44:27.858 DEBUG:teuthology.orchestra.run.trial048:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T21:44:27.879 INFO:teuthology.orchestra.run.trial048.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T21:44:27.880 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T21:44:27.880 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T21:44:27.883 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:44:28.024 DEBUG:teuthology.orchestra.run.trial104:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T21:44:28.046 INFO:teuthology.orchestra.run.trial104.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T21:44:28.046 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T21:44:28.046 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T21:44:28.048 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-22T21:44:28.049 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:28.049 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T21:44:28.075 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:28.075 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T21:44:28.112 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-22T21:44:28.113 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:28.113 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T21:44:28.140 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T21:44:28.205 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:28.205 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T21:44:28.231 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T21:44:28.297 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-22T21:44:28.297 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:28.298 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T21:44:28.324 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T21:44:28.388 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:28.389 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T21:44:28.413 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T21:44:28.477 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-22T21:44:28.478 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:28.478 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T21:44:28.503 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T21:44:28.569 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:28.570 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T21:44:28.596 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T21:44:28.659 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T21:44:28.770 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-22T21:44:28.771 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod 777 /var/log/ceph 2026-02-22T21:44:28.772 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod 777 /var/log/ceph 2026-02-22T21:44:28.796 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-22T21:44:28.796 DEBUG:teuthology.orchestra.run.trial048:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T21:44:28.836 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T21:44:28.860 INFO:tasks.ceph:Creating extra log directories... 2026-02-22T21:44:28.860 DEBUG:teuthology.orchestra.run.trial048:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T21:44:28.900 DEBUG:teuthology.orchestra.run.trial104:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T21:44:28.942 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-22T21:44:28.942 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T21:44:28.943 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-workunit-policy-none}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64096', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial048.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJJMis5tQRplKOa2XUKywCRJIwG5KxOlyoxvtJs22BJAkAwOpZt/4EmbZE9DgzvNfdZLt0tfEFyJGaAph1qzPS0=', 'trial104.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMrJCFPlCkMRhmAELVBPerDyM3cOH9yMWaJ5BdKzAXf9e5uCF8/sM2a8qcWXicwuezEoql2WVcFQ7Alnj+tRo3c='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T21:44:28.944 DEBUG:teuthology.orchestra.run.trial048:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T21:44:28.965 DEBUG:teuthology.orchestra.run.trial104:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T21:44:28.984 DEBUG:teuthology.orchestra.run.trial048:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:44:29.023 DEBUG:teuthology.orchestra.run.trial104:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:44:29.048 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:29.048 DEBUG:teuthology.orchestra.run.trial048:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T21:44:29.102 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T21:44:29.102 DEBUG:teuthology.orchestra.run.trial048:> stat /dev/vg_nvme/lv_1 2026-02-22T21:44:29.159 INFO:teuthology.orchestra.run.trial048.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T21:44:29.159 INFO:teuthology.orchestra.run.trial048.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:44:29.160 INFO:teuthology.orchestra.run.trial048.stdout:Device: 6h/6d Inode: 1294 Links: 1 2026-02-22T21:44:29.160 INFO:teuthology.orchestra.run.trial048.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:44:29.160 INFO:teuthology.orchestra.run.trial048.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:44:29.160 INFO:teuthology.orchestra.run.trial048.stdout:Access: 2026-02-22 21:44:26.051281348 +0000 2026-02-22T21:44:29.160 INFO:teuthology.orchestra.run.trial048.stdout:Modify: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.160 INFO:teuthology.orchestra.run.trial048.stdout:Change: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.160 INFO:teuthology.orchestra.run.trial048.stdout: Birth: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.160 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T21:44:29.223 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records in 2026-02-22T21:44:29.223 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records out 2026-02-22T21:44:29.224 INFO:teuthology.orchestra.run.trial048.stderr:512 bytes copied, 0.000149266 s, 3.4 MB/s 2026-02-22T21:44:29.225 DEBUG:teuthology.orchestra.run.trial048:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T21:44:29.284 DEBUG:teuthology.orchestra.run.trial048:> stat /dev/vg_nvme/lv_2 2026-02-22T21:44:29.340 INFO:teuthology.orchestra.run.trial048.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T21:44:29.340 INFO:teuthology.orchestra.run.trial048.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:44:29.340 INFO:teuthology.orchestra.run.trial048.stdout:Device: 6h/6d Inode: 1307 Links: 1 2026-02-22T21:44:29.340 INFO:teuthology.orchestra.run.trial048.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:44:29.341 INFO:teuthology.orchestra.run.trial048.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:44:29.341 INFO:teuthology.orchestra.run.trial048.stdout:Access: 2026-02-22 21:44:26.051281348 +0000 2026-02-22T21:44:29.341 INFO:teuthology.orchestra.run.trial048.stdout:Modify: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.341 INFO:teuthology.orchestra.run.trial048.stdout:Change: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.341 INFO:teuthology.orchestra.run.trial048.stdout: Birth: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.341 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T21:44:29.404 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records in 2026-02-22T21:44:29.404 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records out 2026-02-22T21:44:29.404 INFO:teuthology.orchestra.run.trial048.stderr:512 bytes copied, 0.000136572 s, 3.7 MB/s 2026-02-22T21:44:29.406 DEBUG:teuthology.orchestra.run.trial048:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T21:44:29.464 DEBUG:teuthology.orchestra.run.trial048:> stat /dev/vg_nvme/lv_3 2026-02-22T21:44:29.521 INFO:teuthology.orchestra.run.trial048.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T21:44:29.521 INFO:teuthology.orchestra.run.trial048.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:44:29.521 INFO:teuthology.orchestra.run.trial048.stdout:Device: 6h/6d Inode: 1298 Links: 1 2026-02-22T21:44:29.521 INFO:teuthology.orchestra.run.trial048.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:44:29.521 INFO:teuthology.orchestra.run.trial048.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:44:29.521 INFO:teuthology.orchestra.run.trial048.stdout:Access: 2026-02-22 21:44:26.051281348 +0000 2026-02-22T21:44:29.521 INFO:teuthology.orchestra.run.trial048.stdout:Modify: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.522 INFO:teuthology.orchestra.run.trial048.stdout:Change: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.522 INFO:teuthology.orchestra.run.trial048.stdout: Birth: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.522 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T21:44:29.585 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records in 2026-02-22T21:44:29.585 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records out 2026-02-22T21:44:29.585 INFO:teuthology.orchestra.run.trial048.stderr:512 bytes copied, 0.000168222 s, 3.0 MB/s 2026-02-22T21:44:29.586 DEBUG:teuthology.orchestra.run.trial048:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T21:44:29.645 DEBUG:teuthology.orchestra.run.trial048:> stat /dev/vg_nvme/lv_4 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout:Device: 6h/6d Inode: 1302 Links: 1 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout:Access: 2026-02-22 21:44:26.052281318 +0000 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout:Modify: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.702 INFO:teuthology.orchestra.run.trial048.stdout:Change: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.703 INFO:teuthology.orchestra.run.trial048.stdout: Birth: 2026-02-22 21:43:26.045095595 +0000 2026-02-22T21:44:29.703 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T21:44:29.765 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records in 2026-02-22T21:44:29.765 INFO:teuthology.orchestra.run.trial048.stderr:1+0 records out 2026-02-22T21:44:29.765 INFO:teuthology.orchestra.run.trial048.stderr:512 bytes copied, 0.000147012 s, 3.5 MB/s 2026-02-22T21:44:29.767 DEBUG:teuthology.orchestra.run.trial048:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T21:44:29.824 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:44:29.825 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial048.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:44:29.825 INFO:tasks.ceph:Generating config... 2026-02-22T21:44:29.829 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T21:44:29.829 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T21:44:29.829 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T21:44:29.830 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T21:44:29.830 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2026-02-22T21:44:29.830 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:29.830 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:29.830 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:29.830 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:29.831 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:29.831 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:29.831 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:29.831 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:29.831 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:29.832 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:29.832 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:29.832 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:29.832 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:29.832 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:29.832 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T21:44:29.833 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T21:44:29.833 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T21:44:29.833 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T21:44:29.833 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T21:44:29.833 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T21:44:29.833 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T21:44:29.833 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T21:44:29.833 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T21:44:29.833 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T21:44:29.833 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T21:44:29.834 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T21:44:29.835 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-22T21:44:29.835 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-22T21:44:29.898 INFO:teuthology.orchestra.run.trial048.stdout:creating /etc/ceph/cluster1.keyring 2026-02-22T21:44:29.901 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-22T21:44:29.977 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T21:44:30.039 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.48')] 2026-02-22T21:44:30.040 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.48', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T21:44:30.042 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:30.042 DEBUG:teuthology.orchestra.run.trial048:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:44:30.095 DEBUG:teuthology.orchestra.run.trial048:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.48 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:30.167 INFO:teuthology.orchestra.run.trial048.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:30.167 INFO:teuthology.orchestra.run.trial048.stdout:monmaptool: generated fsid b0936238-8de0-4459-9a27-b7bc50f582a5 2026-02-22T21:44:30.167 INFO:teuthology.orchestra.run.trial048.stdout:setting min_mon_release = quincy 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:epoch 0 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:fsid b0936238-8de0-4459-9a27-b7bc50f582a5 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:last_changed 2026-02-22T21:44:30.166478+0000 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:created 2026-02-22T21:44:30.166478+0000 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:min_mon_release 17 (quincy) 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:election_strategy: 1 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:0: [v2:10.20.193.48:3300/0,v1:10.20.193.48:6789/0] mon.a 2026-02-22T21:44:30.168 INFO:teuthology.orchestra.run.trial048.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-22T21:44:30.170 DEBUG:teuthology.orchestra.run.trial048:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:44:30.228 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID b0936238-8de0-4459-9a27-b7bc50f582a5... 2026-02-22T21:44:30.229 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T21:44:30.270 DEBUG:teuthology.orchestra.run.trial104:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T21:44:30.308 INFO:teuthology.orchestra.run.trial048.stdout:[global] 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: chdir = "" 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: auth supported = cephx 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: filestore xattr use omap = true 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: mon clock drift allowed = 1.000 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.309 INFO:teuthology.orchestra.run.trial048.stdout: osd crush chooseleaf type = 0 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: auth debug = true 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: ms die on old message = true 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: ms die on bug = true 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: mon pg warn max object skew = 0 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:44:30.310 INFO:teuthology.orchestra.run.trial048.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: osd pool default size = 2 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon osd allow primary affinity = true 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon osd allow pg remap = true 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on no sortbitwise = false 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on too few osds = false 2026-02-22T21:44:30.311 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: mon_allow_pool_size_one = true 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: osd default data pool replay window = 5 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: mon allow pool delete = true 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: mon cluster log file level = debug 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: debug asserts on shutdown = true 2026-02-22T21:44:30.312 INFO:teuthology.orchestra.run.trial048.stdout: mon health detail to clog = false 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: mon host = 10.20.193.48 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: mon client directed command retry = 5 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on pool no app = False 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: ms inject socket failures = 5000 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: fsid = b0936238-8de0-4459-9a27-b7bc50f582a5 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout:[osd] 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: osd journal size = 100 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: osd scrub load threshold = 5.0 2026-02-22T21:44:30.313 INFO:teuthology.orchestra.run.trial048.stdout: osd scrub max interval = 600 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd mclock skip benchmark = true 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd recover clone overlap = true 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd recovery max chunk = 1048576 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd debug shutdown = true 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd debug op order = true 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd debug verify stray on activate = true 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd debug trim objects = true 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd open classes on start = true 2026-02-22T21:44:30.314 INFO:teuthology.orchestra.run.trial048.stdout: osd debug pg log writeout = true 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: osd map max advance = 10 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: journal zero on create = true 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: filestore apply finisher threads = 3 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: bdev debug aio = true 2026-02-22T21:44:30.315 INFO:teuthology.orchestra.run.trial048.stdout: osd debug misdirected ops = true 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: bdev async discard = True 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: bdev enable discard = True 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: bluestore allocator = bitmap 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: bluestore block size = 96636764160 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: bluestore fsck on mount = True 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: debug bluefs = 1/20 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: debug bluestore = 1/20 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: debug ms = 1 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: debug osd = 20 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: debug rocksdb = 4/10 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: mon osd full ratio = 0.9 2026-02-22T21:44:30.316 INFO:teuthology.orchestra.run.trial048.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: osd objectstore = bluestore 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout:[mgr] 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: debug ms = 1 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: debug mgr = 20 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: debug mon = 20 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: debug auth = 20 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: mgr/telemetry/nag = false 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.317 INFO:teuthology.orchestra.run.trial048.stdout:[mon] 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: debug ms = 1 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: debug mon = 20 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: debug paxos = 20 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: debug auth = 20 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: mon data avail warn = 5 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: mon mgr mkfs grace = 240 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: mon osd reporter subtree level = osd 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: mon osd prime pg temp = true 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.318 INFO:teuthology.orchestra.run.trial048.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: # 1m isn't quite enough 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:30.319 INFO:teuthology.orchestra.run.trial048.stdout:[client] 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: rgw cache enabled = true 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: rgw enable ops log = true 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: rgw enable usage log = true 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: debug rbd = 20 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: debug rbd_mirror = 15 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: log to stderr = False 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: rbd default features = 125 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: rbd mirror image policy type = none 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout:[mon.a] 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.0] 2026-02-22T21:44:30.320 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.1] 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.2] 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.3] 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.4] 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.321 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.322 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.5] 2026-02-22T21:44:30.322 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.322 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.322 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.6] 2026-02-22T21:44:30.322 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.322 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.324 INFO:teuthology.orchestra.run.trial104.stdout:[global] 2026-02-22T21:44:30.324 INFO:teuthology.orchestra.run.trial104.stdout: chdir = "" 2026-02-22T21:44:30.324 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:44:30.324 INFO:teuthology.orchestra.run.trial104.stdout: auth supported = cephx 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: filestore xattr use omap = true 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: mon clock drift allowed = 1.000 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: osd crush chooseleaf type = 0 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: auth debug = true 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: ms die on old message = true 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: ms die on bug = true 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.325 INFO:teuthology.orchestra.run.trial104.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: mon pg warn max object skew = 0 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: osd pool default size = 2 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: mon osd allow primary affinity = true 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: mon osd allow pg remap = true 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:44:30.326 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on no sortbitwise = false 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on too few osds = false 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: mon_allow_pool_size_one = true 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: osd default data pool replay window = 5 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.327 INFO:teuthology.orchestra.run.trial104.stdout: mon allow pool delete = true 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: mon cluster log file level = debug 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: debug asserts on shutdown = true 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: mon health detail to clog = false 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: mon host = 10.20.193.48 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: mon client directed command retry = 5 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on pool no app = False 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: ms inject socket failures = 5000 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: fsid = b0936238-8de0-4459-9a27-b7bc50f582a5 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout:[osd] 2026-02-22T21:44:30.328 INFO:teuthology.orchestra.run.trial104.stdout: osd journal size = 100 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd scrub load threshold = 5.0 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd scrub max interval = 600 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd mclock skip benchmark = true 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd recover clone overlap = true 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd recovery max chunk = 1048576 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd debug shutdown = true 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd debug op order = true 2026-02-22T21:44:30.329 INFO:teuthology.orchestra.run.trial104.stdout: osd debug verify stray on activate = true 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: osd debug trim objects = true 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: osd open classes on start = true 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: osd debug pg log writeout = true 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: osd map max advance = 10 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: journal zero on create = true 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:44:30.330 INFO:teuthology.orchestra.run.trial104.stdout: filestore apply finisher threads = 3 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: bdev debug aio = true 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: osd debug misdirected ops = true 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: bdev async discard = True 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: bdev enable discard = True 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: bluestore allocator = bitmap 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: bluestore block size = 96636764160 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: bluestore fsck on mount = True 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: debug bluefs = 1/20 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: debug bluestore = 1/20 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: debug ms = 1 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: debug osd = 20 2026-02-22T21:44:30.331 INFO:teuthology.orchestra.run.trial104.stdout: debug rocksdb = 4/10 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: mon osd full ratio = 0.9 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: osd objectstore = bluestore 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout:[mgr] 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: debug ms = 1 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: debug mgr = 20 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: debug mon = 20 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: debug auth = 20 2026-02-22T21:44:30.332 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: mgr/telemetry/nag = false 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout:[mon] 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: debug ms = 1 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: debug mon = 20 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: debug paxos = 20 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: debug auth = 20 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: mon data avail warn = 5 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: mon mgr mkfs grace = 240 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:44:30.333 INFO:teuthology.orchestra.run.trial104.stdout: mon osd reporter subtree level = osd 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: mon osd prime pg temp = true 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.334 INFO:teuthology.orchestra.run.trial104.stdout: # 1m isn't quite enough 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout:[client] 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: rgw cache enabled = true 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: rgw enable ops log = true 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: rgw enable usage log = true 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: debug rbd = 20 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: debug rbd_mirror = 15 2026-02-22T21:44:30.335 INFO:teuthology.orchestra.run.trial104.stdout: log to stderr = False 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: rbd default features = 125 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: rbd mirror image policy type = none 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout:[mon.a] 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.0] 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.1] 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.2] 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.336 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.3] 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.4] 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.5] 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.6] 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:44:30.337 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:44:30.338 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-22T21:44:30.338 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-22T21:44:30.407 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T21:44:30.408 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:30.408 DEBUG:teuthology.orchestra.run.trial048:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-22T21:44:30.464 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:30.464 DEBUG:teuthology.orchestra.run.trial048:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-22T21:44:30.521 INFO:tasks.ceph:Sending monmap to node ubuntu@trial048.front.sepia.ceph.com 2026-02-22T21:44:30.521 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:30.522 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T21:44:30.522 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T21:44:30.594 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:30.595 DEBUG:teuthology.orchestra.run.trial048:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:30.652 INFO:tasks.ceph:Sending monmap to node ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:44:30.652 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:30.653 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T21:44:30.653 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T21:44:30.687 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:30.687 DEBUG:teuthology.orchestra.run.trial104:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:30.743 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T21:44:30.743 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T21:44:30.743 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T21:44:30.787 INFO:teuthology.orchestra.run.trial048.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T21:44:30.789 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T21:44:30.789 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T21:44:30.790 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-22T21:44:30.863 INFO:teuthology.orchestra.run.trial048.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-22T21:44:30.874 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T21:44:30.910 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T21:44:30.921 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T21:44:30.995 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T21:44:31.006 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T21:44:31.079 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T21:44:31.091 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T21:44:31.164 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T21:44:31.175 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T21:44:31.250 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T21:44:31.261 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T21:44:31.334 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T21:44:31.345 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T21:44:31.419 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T21:44:31.430 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T21:44:31.503 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T21:44:31.514 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T21:44:31.514 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial048.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:44:31.515 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-22T21:44:31.540 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:44:31.540 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-22T21:44:31.540 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial048.front.sepia.ceph.com 2026-02-22T21:44:31.541 DEBUG:teuthology.orchestra.run.trial048:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T21:44:31.605 INFO:teuthology.orchestra.run.trial048.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:44:31.605 INFO:teuthology.orchestra.run.trial048.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:44:31.606 INFO:teuthology.orchestra.run.trial048.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:44:32.176 INFO:teuthology.orchestra.run.trial048.stdout:Discarding blocks...Done. 2026-02-22T21:44:32.178 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial048.front.sepia.ceph.com -o noatime 2026-02-22T21:44:32.178 DEBUG:teuthology.orchestra.run.trial048:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-22T21:44:32.247 DEBUG:teuthology.orchestra.run.trial048:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-22T21:44:32.318 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-22T21:44:32.382 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:44:32.382 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-22T21:44:32.382 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial048.front.sepia.ceph.com 2026-02-22T21:44:32.382 DEBUG:teuthology.orchestra.run.trial048:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:44:32.450 INFO:teuthology.orchestra.run.trial048.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:44:32.451 INFO:teuthology.orchestra.run.trial048.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:44:32.451 INFO:teuthology.orchestra.run.trial048.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:44:33.010 INFO:teuthology.orchestra.run.trial048.stdout:Discarding blocks...Done. 2026-02-22T21:44:33.012 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial048.front.sepia.ceph.com -o noatime 2026-02-22T21:44:33.012 DEBUG:teuthology.orchestra.run.trial048:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-22T21:44:33.081 DEBUG:teuthology.orchestra.run.trial048:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-22T21:44:33.147 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-22T21:44:33.213 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:44:33.213 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-22T21:44:33.213 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial048.front.sepia.ceph.com 2026-02-22T21:44:33.213 DEBUG:teuthology.orchestra.run.trial048:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T21:44:33.277 INFO:teuthology.orchestra.run.trial048.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:44:33.277 INFO:teuthology.orchestra.run.trial048.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:44:33.277 INFO:teuthology.orchestra.run.trial048.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:44:33.278 INFO:teuthology.orchestra.run.trial048.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:44:33.278 INFO:teuthology.orchestra.run.trial048.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:44:33.278 INFO:teuthology.orchestra.run.trial048.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:44:33.278 INFO:teuthology.orchestra.run.trial048.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:44:33.278 INFO:teuthology.orchestra.run.trial048.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:44:33.278 INFO:teuthology.orchestra.run.trial048.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:44:33.278 INFO:teuthology.orchestra.run.trial048.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:44:33.832 INFO:teuthology.orchestra.run.trial048.stdout:Discarding blocks...Done. 2026-02-22T21:44:33.834 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial048.front.sepia.ceph.com -o noatime 2026-02-22T21:44:33.835 DEBUG:teuthology.orchestra.run.trial048:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-22T21:44:33.902 DEBUG:teuthology.orchestra.run.trial048:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-22T21:44:33.970 DEBUG:teuthology.orchestra.run.trial048:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:34.048 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:34.046+0000 7fbec18ed780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-22T21:44:34.048 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:34.046+0000 7fbec18ed780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-22T21:44:34.048 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:34.046+0000 7fbec18ed780 -1 bdev(0x5643b6933800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-22T21:44:34.048 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:34.046+0000 7fbec18ed780 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-22T21:44:37.935 DEBUG:teuthology.orchestra.run.trial048:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-22T21:44:38.000 DEBUG:teuthology.orchestra.run.trial048:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:38.076 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:38.074+0000 7f84cc5a3780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-22T21:44:38.076 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:38.074+0000 7f84cc5a3780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-22T21:44:38.076 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:38.074+0000 7f84cc5a3780 -1 bdev(0x56153024f800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-22T21:44:38.076 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:38.074+0000 7f84cc5a3780 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-22T21:44:41.962 DEBUG:teuthology.orchestra.run.trial048:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-22T21:44:42.027 DEBUG:teuthology.orchestra.run.trial048:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:42.107 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:42.104+0000 7f524394f780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-22T21:44:42.108 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:42.105+0000 7f524394f780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-22T21:44:42.108 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:42.105+0000 7f524394f780 -1 bdev(0x555f18d21800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-22T21:44:42.108 INFO:teuthology.orchestra.run.trial048.stderr:2026-02-22T21:44:42.105+0000 7f524394f780 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-22T21:44:45.999 DEBUG:teuthology.orchestra.run.trial048:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-22T21:44:46.064 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T21:44:46.065 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:46.065 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-22T21:44:46.129 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:46.130 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-22T21:44:46.195 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:46.195 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-22T21:44:46.258 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:46.258 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-22T21:44:46.321 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:46.321 DEBUG:teuthology.orchestra.run.trial048:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-22T21:44:46.378 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.378 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-22T21:44:46.396 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.396 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-22T21:44:46.452 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.452 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-22T21:44:46.510 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.510 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-22T21:44:46.566 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.566 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-22T21:44:46.621 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.621 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-22T21:44:46.679 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.679 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-22T21:44:46.734 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:44:46.735 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-22T21:44:46.792 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T21:44:46.793 DEBUG:teuthology.orchestra.run.trial048:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-22T21:44:46.817 INFO:teuthology.orchestra.run.trial048.stdout:[mgr.x] 2026-02-22T21:44:46.817 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC+eJtpYTDhLhAA9Z6r6NF3658WwYvBPpn0rQ== 2026-02-22T21:44:46.817 INFO:teuthology.orchestra.run.trial048.stdout:[osd.0] 2026-02-22T21:44:46.817 INFO:teuthology.orchestra.run.trial048.stdout: key = AQDCeJtphdjOAhAAzgiwqODuJ5XyR2e9YCFE0A== 2026-02-22T21:44:46.817 INFO:teuthology.orchestra.run.trial048.stdout:[osd.1] 2026-02-22T21:44:46.817 INFO:teuthology.orchestra.run.trial048.stdout: key = AQDGeJtpdz2GBBAAfMYHFxcn5bxtOiuw0xBcaw== 2026-02-22T21:44:46.817 INFO:teuthology.orchestra.run.trial048.stdout:[osd.2] 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout: key = AQDKeJtphP5ZBhAAoshZIs4oQ0WGOOO7IMnFZQ== 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout:[client.0] 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC+eJtpXYVyMxAA4x1xmSDZgJMwP9cetpaO8A== 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror] 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC+eJtpqVY9NhAAziZqDFCXaT+5A1zpHmFb1g== 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.0] 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC+eJtpdzJGOxAAOr2AERBYLBCmJN6Ax0mq8w== 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.1] 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC/eJtpFnG5BBAARvbizJ3RD7ciUL53LGflKA== 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.2] 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC/eJtp+uG7CRAATEOikjR2MPx5Gm6R3b1cIQ== 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.3] 2026-02-22T21:44:46.818 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC/eJtpNLfgDhAAdaTWhiicBckydpszzf7VGA== 2026-02-22T21:44:46.819 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.4] 2026-02-22T21:44:46.819 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC/eJtp4DfoExAAJJDaDhCHNobsqMVC3HAknw== 2026-02-22T21:44:46.819 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.5] 2026-02-22T21:44:46.819 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC/eJtp+QPvGBAAm2ZfFewDX+H2VwNL1lps+w== 2026-02-22T21:44:46.819 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.6] 2026-02-22T21:44:46.819 INFO:teuthology.orchestra.run.trial048.stdout: key = AQC/eJtpLRn2HRAA6ZX1Dx/Y5nIbl8upDlbPfw== 2026-02-22T21:44:46.820 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T21:44:46.901 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:44:46.983 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:44:47.062 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:44:47.142 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.223 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.304 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.387 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.467 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.548 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.629 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.709 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.790 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:44:47.871 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T21:44:47.871 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-22T21:44:47.934 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-22T21:44:48.022 DEBUG:teuthology.orchestra.run.trial048:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-22T21:44:48.047 DEBUG:teuthology.orchestra.run.trial048:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T21:44:48.104 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-22T21:44:48.104 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-22T21:44:48.105 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-22T21:44:48.147 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-22T21:44:48.147 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-22T21:44:48.147 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-22T21:44:48.147 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-22T21:44:48.150 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-22T21:44:48.150 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T21:44:48.150 DEBUG:teuthology.orchestra.run.trial048:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-22T21:44:48.254 INFO:teuthology.orchestra.run.trial048.stderr:adjusted tunables profile to default 2026-02-22T21:44:48.262 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T21:44:48.262 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-22T21:44:48.262 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:48.262 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-22T21:44:48.287 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:48.288 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-22T21:44:48.352 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:44:48.352 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-22T21:44:48.416 DEBUG:teuthology.orchestra.run.trial048:> sudo ceph --cluster cluster1 osd new 13807f88-98fa-4d56-b03f-7a5409e66ae9 0 2026-02-22T21:44:48.560 INFO:teuthology.orchestra.run.trial048.stdout:0 2026-02-22T21:44:48.569 DEBUG:teuthology.orchestra.run.trial048:> sudo ceph --cluster cluster1 osd new b010d8db-2fbc-495e-bb8d-60a37b18f394 1 2026-02-22T21:44:48.673 INFO:teuthology.orchestra.run.trial048.stdout:1 2026-02-22T21:44:48.681 DEBUG:teuthology.orchestra.run.trial048:> sudo ceph --cluster cluster1 osd new 9121920a-57f3-4109-b3a3-b027b2d3ac5d 2 2026-02-22T21:44:48.760 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:48.759+0000 7f83f2613140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T21:44:48.786 INFO:teuthology.orchestra.run.trial048.stdout:2 2026-02-22T21:44:48.795 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-22T21:44:48.796 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-22T21:44:48.798 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-22T21:44:48.798 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-22T21:44:48.799 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-22T21:44:48.801 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-22T21:44:48.801 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-22T21:44:48.801 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-22T21:44:48.803 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-22T21:44:48.804 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:44:48.885 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:48.884+0000 7f83f2613140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T21:44:48.907 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:44:48.907 INFO:teuthology.orchestra.run.trial048.stdout:{"epoch":5,"fsid":"b0936238-8de0-4459-9a27-b7bc50f582a5","created":"2026-02-22T21:44:48.208030+0000","modified":"2026-02-22T21:44:48.785911+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:44:48.785911+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"13807f88-98fa-4d56-b03f-7a5409e66ae9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b010d8db-2fbc-495e-bb8d-60a37b18f394","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9121920a-57f3-4109-b3a3-b027b2d3ac5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:44:48.913 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-22T21:44:48.914 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T21:44:48.914 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:48.911+0000 7f83f2613140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T21:44:48.946 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:48.945+0000 7f83f2613140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T21:44:48.969 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:48.968+0000 7f83f2613140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T21:44:48.996 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:48.994+0000 7f83f2613140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T21:44:49.051 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.050+0000 7f83f2613140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T21:44:49.112 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T21:44:49.112 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T21:44:49.112 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr: from numpy import show_config as show_numpy_config 2026-02-22T21:44:49.116 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.115+0000 7f83f2613140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T21:44:49.141 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.140+0000 7f83f2613140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T21:44:49.197 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.195+0000 7f83f2613140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T21:44:49.216 DEBUG:teuthology.orchestra.run.trial048:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:44:49.240 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.239+0000 7f83f2613140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T21:44:49.267 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.265+0000 7f83f2613140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T21:44:49.292 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.290+0000 7f83f2613140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T21:44:49.310 INFO:teuthology.misc.health.trial048.stdout: 2026-02-22T21:44:49.310 INFO:teuthology.misc.health.trial048.stdout:{"epoch":5,"fsid":"b0936238-8de0-4459-9a27-b7bc50f582a5","created":"2026-02-22T21:44:48.208030+0000","modified":"2026-02-22T21:44:48.785911+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:44:48.785911+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"13807f88-98fa-4d56-b03f-7a5409e66ae9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b010d8db-2fbc-495e-bb8d-60a37b18f394","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9121920a-57f3-4109-b3a3-b027b2d3ac5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:44:49.316 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:44:49.317 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.316+0000 7f83f2613140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T21:44:49.341 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.340+0000 7f83f2613140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T21:44:49.538 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.536+0000 7f83f2613140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T21:44:49.593 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.591+0000 7f83f2613140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T21:44:49.615 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.614+0000 7f83f2613140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T21:44:49.638 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.637+0000 7f83f2613140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T21:44:49.692 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.690+0000 7f83f2613140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:44:49.712 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.711+0000 7f83f2613140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T21:44:49.765 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:49.763+0000 7f83f2613140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:44:50.164 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:50.162+0000 7f83f2613140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T21:44:50.186 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:50.184+0000 7f83f2613140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T21:44:50.525 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:50.524+0000 7f83f2613140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T21:44:50.718 INFO:tasks.ceph.cluster1.osd.0.trial048.stderr:2026-02-22T21:44:50.716+0000 7f7ec85b2780 -1 Falling back to public interface 2026-02-22T21:44:50.744 INFO:tasks.ceph.cluster1.osd.1.trial048.stderr:2026-02-22T21:44:50.742+0000 7f43672a4780 -1 Falling back to public interface 2026-02-22T21:44:50.757 INFO:tasks.ceph.cluster1.osd.2.trial048.stderr:2026-02-22T21:44:50.755+0000 7fa62e2e7780 -1 Falling back to public interface 2026-02-22T21:44:51.531 INFO:tasks.ceph.cluster1.mgr.x.trial048.stderr:2026-02-22T21:44:51.529+0000 7f830ef4e640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T21:44:55.193 INFO:tasks.ceph.cluster1.osd.0.trial048.stderr:2026-02-22T21:44:55.191+0000 7f7ec85b2780 -1 osd.0 0 log_to_monitors true 2026-02-22T21:44:55.231 INFO:tasks.ceph.cluster1.osd.1.trial048.stderr:2026-02-22T21:44:55.230+0000 7f43672a4780 -1 osd.1 0 log_to_monitors true 2026-02-22T21:44:55.273 INFO:tasks.ceph.cluster1.osd.2.trial048.stderr:2026-02-22T21:44:55.271+0000 7fa62e2e7780 -1 osd.2 0 log_to_monitors true 2026-02-22T21:44:55.620 DEBUG:teuthology.orchestra.run.trial048:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:44:55.768 INFO:teuthology.misc.health.trial048.stdout: 2026-02-22T21:44:55.769 INFO:teuthology.misc.health.trial048.stdout:{"epoch":6,"fsid":"b0936238-8de0-4459-9a27-b7bc50f582a5","created":"2026-02-22T21:44:48.208030+0000","modified":"2026-02-22T21:44:55.244432+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:44:48.785911+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"13807f88-98fa-4d56-b03f-7a5409e66ae9","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b010d8db-2fbc-495e-bb8d-60a37b18f394","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"9121920a-57f3-4109-b3a3-b027b2d3ac5d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:44:55.775 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:44:56.262 INFO:tasks.ceph.cluster1.osd.0.trial048.stderr:2026-02-22T21:44:56.261+0000 7f7ec451d640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T21:44:56.262 INFO:tasks.ceph.cluster1.osd.1.trial048.stderr:2026-02-22T21:44:56.261+0000 7f4363211640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T21:44:56.264 INFO:tasks.ceph.cluster1.osd.1.trial048.stderr:2026-02-22T21:44:56.262+0000 7f435b044640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:44:56.264 INFO:tasks.ceph.cluster1.osd.0.trial048.stderr:2026-02-22T21:44:56.263+0000 7f7ebc350640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:44:57.261 INFO:tasks.ceph.cluster1.osd.2.trial048.stderr:2026-02-22T21:44:57.260+0000 7fa62a252640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T21:44:57.263 INFO:tasks.ceph.cluster1.osd.2.trial048.stderr:2026-02-22T21:44:57.262+0000 7fa622085640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:45:02.078 DEBUG:teuthology.orchestra.run.trial048:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:45:02.231 INFO:teuthology.misc.health.trial048.stdout: 2026-02-22T21:45:02.231 INFO:teuthology.misc.health.trial048.stdout:{"epoch":12,"fsid":"b0936238-8de0-4459-9a27-b7bc50f582a5","created":"2026-02-22T21:44:48.208030+0000","modified":"2026-02-22T21:45:01.255561+0000","last_up_change":"2026-02-22T21:44:58.250093+0000","last_in_change":"2026-02-22T21:44:48.785911+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:44:58.539685+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"13807f88-98fa-4d56-b03f-7a5409e66ae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6802","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6803","nonce":2906375031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6804","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6805","nonce":2906375031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6808","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6809","nonce":2906375031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6806","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6807","nonce":2906375031}]},"public_addr":"10.20.193.48:6803/2906375031","cluster_addr":"10.20.193.48:6805/2906375031","heartbeat_back_addr":"10.20.193.48:6809/2906375031","heartbeat_front_addr":"10.20.193.48:6807/2906375031","state":["exists","up"]},{"osd":1,"uuid":"b010d8db-2fbc-495e-bb8d-60a37b18f394","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6810","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6811","nonce":2771328344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6812","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6813","nonce":2771328344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6816","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6817","nonce":2771328344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6814","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6815","nonce":2771328344}]},"public_addr":"10.20.193.48:6811/2771328344","cluster_addr":"10.20.193.48:6813/2771328344","heartbeat_back_addr":"10.20.193.48:6817/2771328344","heartbeat_front_addr":"10.20.193.48:6815/2771328344","state":["exists","up"]},{"osd":2,"uuid":"9121920a-57f3-4109-b3a3-b027b2d3ac5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6818","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6819","nonce":3646627341}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6820","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6821","nonce":3646627341}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6824","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6825","nonce":3646627341}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6822","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6823","nonce":3646627341}]},"public_addr":"10.20.193.48:6819/3646627341","cluster_addr":"10.20.193.48:6821/3646627341","heartbeat_back_addr":"10.20.193.48:6825/3646627341","heartbeat_front_addr":"10.20.193.48:6823/3646627341","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.239903+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.217985+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.265063+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:02.239 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T21:45:02.239 INFO:tasks.ceph:Creating RBD pool 2026-02-22T21:45:02.239 DEBUG:teuthology.orchestra.run.trial048:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-22T21:45:03.261 INFO:teuthology.orchestra.run.trial048.stderr:pool 'rbd' created 2026-02-22T21:45:03.272 DEBUG:teuthology.orchestra.run.trial048:> rbd --cluster cluster1 pool init rbd 2026-02-22T21:45:06.274 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-22T21:45:06.275 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-22T21:45:06.275 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-22T21:45:06.276 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-22T21:45:06.276 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T21:45:06.455 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:06.462 INFO:teuthology.orchestra.run.trial048.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6800","nonce":1651990073},{"type":"v1","addr":"10.20.193.48:6801","nonce":1651990073}]},"active_addr":"10.20.193.48:6801/1651990073","active_change":"2026-02-22T21:44:50.525668+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":3409457783}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":1649791880}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":697676395}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":1832401455}]}]} 2026-02-22T21:45:06.466 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-22T21:45:06.467 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-22T21:45:06.467 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:45:06.624 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:06.624 INFO:teuthology.orchestra.run.trial048.stdout:{"epoch":16,"fsid":"b0936238-8de0-4459-9a27-b7bc50f582a5","created":"2026-02-22T21:44:48.208030+0000","modified":"2026-02-22T21:45:06.265596+0000","last_up_change":"2026-02-22T21:44:58.250093+0000","last_in_change":"2026-02-22T21:44:48.785911+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:44:58.539685+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:45:02.395533+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"13807f88-98fa-4d56-b03f-7a5409e66ae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6802","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6803","nonce":2906375031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6804","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6805","nonce":2906375031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6808","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6809","nonce":2906375031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6806","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6807","nonce":2906375031}]},"public_addr":"10.20.193.48:6803/2906375031","cluster_addr":"10.20.193.48:6805/2906375031","heartbeat_back_addr":"10.20.193.48:6809/2906375031","heartbeat_front_addr":"10.20.193.48:6807/2906375031","state":["exists","up"]},{"osd":1,"uuid":"b010d8db-2fbc-495e-bb8d-60a37b18f394","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6810","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6811","nonce":2771328344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6812","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6813","nonce":2771328344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6816","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6817","nonce":2771328344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6814","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6815","nonce":2771328344}]},"public_addr":"10.20.193.48:6811/2771328344","cluster_addr":"10.20.193.48:6813/2771328344","heartbeat_back_addr":"10.20.193.48:6817/2771328344","heartbeat_front_addr":"10.20.193.48:6815/2771328344","state":["exists","up"]},{"osd":2,"uuid":"9121920a-57f3-4109-b3a3-b027b2d3ac5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6818","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6819","nonce":3646627341}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6820","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6821","nonce":3646627341}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6824","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6825","nonce":3646627341}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6822","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6823","nonce":3646627341}]},"public_addr":"10.20.193.48:6819/3646627341","cluster_addr":"10.20.193.48:6821/3646627341","heartbeat_back_addr":"10.20.193.48:6825/3646627341","heartbeat_front_addr":"10.20.193.48:6823/3646627341","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.239903+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.217985+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.265063+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:06.632 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-22T21:45:06.633 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T21:45:06.791 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:06.791 INFO:teuthology.orchestra.run.trial048.stdout:{"epoch":16,"fsid":"b0936238-8de0-4459-9a27-b7bc50f582a5","created":"2026-02-22T21:44:48.208030+0000","modified":"2026-02-22T21:45:06.265596+0000","last_up_change":"2026-02-22T21:44:58.250093+0000","last_in_change":"2026-02-22T21:44:48.785911+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:44:58.539685+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:45:02.395533+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"13807f88-98fa-4d56-b03f-7a5409e66ae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6802","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6803","nonce":2906375031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6804","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6805","nonce":2906375031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6808","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6809","nonce":2906375031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6806","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6807","nonce":2906375031}]},"public_addr":"10.20.193.48:6803/2906375031","cluster_addr":"10.20.193.48:6805/2906375031","heartbeat_back_addr":"10.20.193.48:6809/2906375031","heartbeat_front_addr":"10.20.193.48:6807/2906375031","state":["exists","up"]},{"osd":1,"uuid":"b010d8db-2fbc-495e-bb8d-60a37b18f394","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6810","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6811","nonce":2771328344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6812","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6813","nonce":2771328344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6816","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6817","nonce":2771328344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6814","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6815","nonce":2771328344}]},"public_addr":"10.20.193.48:6811/2771328344","cluster_addr":"10.20.193.48:6813/2771328344","heartbeat_back_addr":"10.20.193.48:6817/2771328344","heartbeat_front_addr":"10.20.193.48:6815/2771328344","state":["exists","up"]},{"osd":2,"uuid":"9121920a-57f3-4109-b3a3-b027b2d3ac5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6818","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6819","nonce":3646627341}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6820","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6821","nonce":3646627341}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6824","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6825","nonce":3646627341}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6822","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6823","nonce":3646627341}]},"public_addr":"10.20.193.48:6819/3646627341","cluster_addr":"10.20.193.48:6821/3646627341","heartbeat_back_addr":"10.20.193.48:6825/3646627341","heartbeat_front_addr":"10.20.193.48:6823/3646627341","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.239903+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.217985+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.265063+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:06.801 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-22T21:45:06.802 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-22T21:45:06.803 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-22T21:45:06.889 INFO:teuthology.orchestra.run.trial048.stdout:34359738372 2026-02-22T21:45:06.890 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T21:45:06.891 INFO:teuthology.orchestra.run.trial048.stdout:38654705667 2026-02-22T21:45:06.891 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T21:45:06.895 INFO:teuthology.orchestra.run.trial048.stdout:34359738372 2026-02-22T21:45:06.895 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T21:45:07.093 INFO:teuthology.orchestra.run.trial048.stdout:38654705666 2026-02-22T21:45:07.099 INFO:teuthology.orchestra.run.trial048.stdout:34359738371 2026-02-22T21:45:07.103 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705666 for osd.2 2026-02-22T21:45:07.109 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.0 2026-02-22T21:45:07.135 INFO:teuthology.orchestra.run.trial048.stdout:34359738371 2026-02-22T21:45:07.144 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.1 2026-02-22T21:45:08.104 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T21:45:08.110 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T21:45:08.144 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T21:45:08.268 INFO:teuthology.orchestra.run.trial048.stdout:38654705666 2026-02-22T21:45:08.279 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705666 for osd.2 2026-02-22T21:45:08.309 INFO:teuthology.orchestra.run.trial048.stdout:34359738371 2026-02-22T21:45:08.312 INFO:teuthology.orchestra.run.trial048.stdout:34359738371 2026-02-22T21:45:08.319 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.0 2026-02-22T21:45:08.323 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.1 2026-02-22T21:45:09.280 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T21:45:09.320 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T21:45:09.324 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T21:45:09.442 INFO:teuthology.orchestra.run.trial048.stdout:38654705667 2026-02-22T21:45:09.453 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705667 got 38654705667 for osd.2 2026-02-22T21:45:09.453 DEBUG:teuthology.parallel:result is None 2026-02-22T21:45:09.484 INFO:teuthology.orchestra.run.trial048.stdout:34359738373 2026-02-22T21:45:09.487 INFO:teuthology.orchestra.run.trial048.stdout:34359738373 2026-02-22T21:45:09.493 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738373 for osd.1 2026-02-22T21:45:09.494 DEBUG:teuthology.parallel:result is None 2026-02-22T21:45:09.498 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738373 for osd.0 2026-02-22T21:45:09.498 DEBUG:teuthology.parallel:result is None 2026-02-22T21:45:09.498 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T21:45:09.498 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T21:45:09.696 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:09.697 INFO:teuthology.orchestra.run.trial048.stderr:dumped all 2026-02-22T21:45:09.705 INFO:teuthology.orchestra.run.trial048.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T21:45:08.529462+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82468,"kb_used_data":1568,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033052,"statfs":{"total":289910292480,"available":289825845248,"internally_reserved":0,"allocated":1605632,"data_stored":1300421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.268550"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268535+0000","last_change":"2026-02-22T21:45:06.268535+0000","last_active":"2026-02-22T21:45:06.268535+0000","last_peered":"2026-02-22T21:45:06.268535+0000","last_clean":"2026-02-22T21:45:06.268535+0000","last_became_active":"2026-02-22T21:45:04.266643+0000","last_became_peered":"2026-02-22T21:45:04.266643+0000","last_unstale":"2026-02-22T21:45:06.268535+0000","last_undegraded":"2026-02-22T21:45:06.268535+0000","last_fullsized":"2026-02-22T21:45:06.268535+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:35.211466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082153,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268006+0000","last_change":"2026-02-22T21:45:06.268006+0000","last_active":"2026-02-22T21:45:06.268006+0000","last_peered":"2026-02-22T21:45:06.268006+0000","last_clean":"2026-02-22T21:45:06.268006+0000","last_became_active":"2026-02-22T21:45:04.266954+0000","last_became_peered":"2026-02-22T21:45:04.266954+0000","last_unstale":"2026-02-22T21:45:06.268006+0000","last_undegraded":"2026-02-22T21:45:06.268006+0000","last_fullsized":"2026-02-22T21:45:06.268006+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:27:29.460040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030903099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268671+0000","last_change":"2026-02-22T21:45:06.268671+0000","last_active":"2026-02-22T21:45:06.268671+0000","last_peered":"2026-02-22T21:45:06.268671+0000","last_clean":"2026-02-22T21:45:06.268671+0000","last_became_active":"2026-02-22T21:45:04.266721+0000","last_became_peered":"2026-02-22T21:45:04.266721+0000","last_unstale":"2026-02-22T21:45:06.268671+0000","last_undegraded":"2026-02-22T21:45:06.268671+0000","last_fullsized":"2026-02-22T21:45:06.268671+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:11:11.390548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094190300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268222+0000","last_change":"2026-02-22T21:45:06.268222+0000","last_active":"2026-02-22T21:45:06.268222+0000","last_peered":"2026-02-22T21:45:06.268222+0000","last_clean":"2026-02-22T21:45:06.268222+0000","last_became_active":"2026-02-22T21:45:04.266897+0000","last_became_peered":"2026-02-22T21:45:04.266897+0000","last_unstale":"2026-02-22T21:45:06.268222+0000","last_undegraded":"2026-02-22T21:45:06.268222+0000","last_fullsized":"2026-02-22T21:45:06.268222+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:10.132872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000509874,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.270287+0000","last_change":"2026-02-22T21:45:06.269931+0000","last_active":"2026-02-22T21:45:06.270287+0000","last_peered":"2026-02-22T21:45:06.270287+0000","last_clean":"2026-02-22T21:45:06.270287+0000","last_became_active":"2026-02-22T21:45:04.266353+0000","last_became_peered":"2026-02-22T21:45:04.266353+0000","last_unstale":"2026-02-22T21:45:06.270287+0000","last_undegraded":"2026-02-22T21:45:06.270287+0000","last_fullsized":"2026-02-22T21:45:06.270287+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:48:09.631675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080603100000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.794946+0000","last_change":"2026-02-22T21:45:06.795048+0000","last_active":"2026-02-22T21:45:06.794946+0000","last_peered":"2026-02-22T21:45:06.794946+0000","last_clean":"2026-02-22T21:45:06.794946+0000","last_became_active":"2026-02-22T21:45:04.266437+0000","last_became_peered":"2026-02-22T21:45:04.266437+0000","last_unstale":"2026-02-22T21:45:06.794946+0000","last_undegraded":"2026-02-22T21:45:06.794946+0000","last_fullsized":"2026-02-22T21:45:06.794946+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:12:37.999670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021661099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.794976+0000","last_change":"2026-02-22T21:45:06.795071+0000","last_active":"2026-02-22T21:45:06.794976+0000","last_peered":"2026-02-22T21:45:06.794976+0000","last_clean":"2026-02-22T21:45:06.794976+0000","last_became_active":"2026-02-22T21:45:04.266381+0000","last_became_peered":"2026-02-22T21:45:04.266381+0000","last_unstale":"2026-02-22T21:45:06.794976+0000","last_undegraded":"2026-02-22T21:45:06.794976+0000","last_fullsized":"2026-02-22T21:45:06.794976+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:59.877077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022258099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268349+0000","last_change":"2026-02-22T21:45:06.268349+0000","last_active":"2026-02-22T21:45:06.268349+0000","last_peered":"2026-02-22T21:45:06.268349+0000","last_clean":"2026-02-22T21:45:06.268349+0000","last_became_active":"2026-02-22T21:45:04.266127+0000","last_became_peered":"2026-02-22T21:45:04.266127+0000","last_unstale":"2026-02-22T21:45:06.268349+0000","last_undegraded":"2026-02-22T21:45:06.268349+0000","last_fullsized":"2026-02-22T21:45:06.268349+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:39:02.311582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065919899999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.267514+0000","last_change":"2026-02-22T21:45:01.134350+0000","last_active":"2026-02-22T21:45:06.267514+0000","last_peered":"2026-02-22T21:45:06.267514+0000","last_clean":"2026-02-22T21:45:06.267514+0000","last_became_active":"2026-02-22T21:45:01.133900+0000","last_became_peered":"2026-02-22T21:45:01.133900+0000","last_unstale":"2026-02-22T21:45:06.267514+0000","last_undegraded":"2026-02-22T21:45:06.267514+0000","last_fullsized":"2026-02-22T21:45:06.267514+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_clean_scrub_stamp":"2026-02-22T21:44:59.253183+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:32:00.074720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705667,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27092,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344748,"statfs":{"total":96636764160,"available":96609021952,"internally_reserved":0,"allocated":131072,"data_stored":37509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:45:09.706 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T21:45:09.865 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:09.865 INFO:teuthology.orchestra.run.trial048.stderr:dumped all 2026-02-22T21:45:09.873 INFO:teuthology.orchestra.run.trial048.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T21:45:08.529462+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82468,"kb_used_data":1568,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033052,"statfs":{"total":289910292480,"available":289825845248,"internally_reserved":0,"allocated":1605632,"data_stored":1300421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.268550"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268535+0000","last_change":"2026-02-22T21:45:06.268535+0000","last_active":"2026-02-22T21:45:06.268535+0000","last_peered":"2026-02-22T21:45:06.268535+0000","last_clean":"2026-02-22T21:45:06.268535+0000","last_became_active":"2026-02-22T21:45:04.266643+0000","last_became_peered":"2026-02-22T21:45:04.266643+0000","last_unstale":"2026-02-22T21:45:06.268535+0000","last_undegraded":"2026-02-22T21:45:06.268535+0000","last_fullsized":"2026-02-22T21:45:06.268535+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:35.211466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082153,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268006+0000","last_change":"2026-02-22T21:45:06.268006+0000","last_active":"2026-02-22T21:45:06.268006+0000","last_peered":"2026-02-22T21:45:06.268006+0000","last_clean":"2026-02-22T21:45:06.268006+0000","last_became_active":"2026-02-22T21:45:04.266954+0000","last_became_peered":"2026-02-22T21:45:04.266954+0000","last_unstale":"2026-02-22T21:45:06.268006+0000","last_undegraded":"2026-02-22T21:45:06.268006+0000","last_fullsized":"2026-02-22T21:45:06.268006+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:27:29.460040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030903099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268671+0000","last_change":"2026-02-22T21:45:06.268671+0000","last_active":"2026-02-22T21:45:06.268671+0000","last_peered":"2026-02-22T21:45:06.268671+0000","last_clean":"2026-02-22T21:45:06.268671+0000","last_became_active":"2026-02-22T21:45:04.266721+0000","last_became_peered":"2026-02-22T21:45:04.266721+0000","last_unstale":"2026-02-22T21:45:06.268671+0000","last_undegraded":"2026-02-22T21:45:06.268671+0000","last_fullsized":"2026-02-22T21:45:06.268671+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:11:11.390548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094190300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268222+0000","last_change":"2026-02-22T21:45:06.268222+0000","last_active":"2026-02-22T21:45:06.268222+0000","last_peered":"2026-02-22T21:45:06.268222+0000","last_clean":"2026-02-22T21:45:06.268222+0000","last_became_active":"2026-02-22T21:45:04.266897+0000","last_became_peered":"2026-02-22T21:45:04.266897+0000","last_unstale":"2026-02-22T21:45:06.268222+0000","last_undegraded":"2026-02-22T21:45:06.268222+0000","last_fullsized":"2026-02-22T21:45:06.268222+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:10.132872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000509874,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.270287+0000","last_change":"2026-02-22T21:45:06.269931+0000","last_active":"2026-02-22T21:45:06.270287+0000","last_peered":"2026-02-22T21:45:06.270287+0000","last_clean":"2026-02-22T21:45:06.270287+0000","last_became_active":"2026-02-22T21:45:04.266353+0000","last_became_peered":"2026-02-22T21:45:04.266353+0000","last_unstale":"2026-02-22T21:45:06.270287+0000","last_undegraded":"2026-02-22T21:45:06.270287+0000","last_fullsized":"2026-02-22T21:45:06.270287+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:48:09.631675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080603100000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.794946+0000","last_change":"2026-02-22T21:45:06.795048+0000","last_active":"2026-02-22T21:45:06.794946+0000","last_peered":"2026-02-22T21:45:06.794946+0000","last_clean":"2026-02-22T21:45:06.794946+0000","last_became_active":"2026-02-22T21:45:04.266437+0000","last_became_peered":"2026-02-22T21:45:04.266437+0000","last_unstale":"2026-02-22T21:45:06.794946+0000","last_undegraded":"2026-02-22T21:45:06.794946+0000","last_fullsized":"2026-02-22T21:45:06.794946+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:12:37.999670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021661099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.794976+0000","last_change":"2026-02-22T21:45:06.795071+0000","last_active":"2026-02-22T21:45:06.794976+0000","last_peered":"2026-02-22T21:45:06.794976+0000","last_clean":"2026-02-22T21:45:06.794976+0000","last_became_active":"2026-02-22T21:45:04.266381+0000","last_became_peered":"2026-02-22T21:45:04.266381+0000","last_unstale":"2026-02-22T21:45:06.794976+0000","last_undegraded":"2026-02-22T21:45:06.794976+0000","last_fullsized":"2026-02-22T21:45:06.794976+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:59.877077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022258099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.268349+0000","last_change":"2026-02-22T21:45:06.268349+0000","last_active":"2026-02-22T21:45:06.268349+0000","last_peered":"2026-02-22T21:45:06.268349+0000","last_clean":"2026-02-22T21:45:06.268349+0000","last_became_active":"2026-02-22T21:45:04.266127+0000","last_became_peered":"2026-02-22T21:45:04.266127+0000","last_unstale":"2026-02-22T21:45:06.268349+0000","last_undegraded":"2026-02-22T21:45:06.268349+0000","last_fullsized":"2026-02-22T21:45:06.268349+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:39:02.311582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065919899999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:06.267514+0000","last_change":"2026-02-22T21:45:01.134350+0000","last_active":"2026-02-22T21:45:06.267514+0000","last_peered":"2026-02-22T21:45:06.267514+0000","last_clean":"2026-02-22T21:45:06.267514+0000","last_became_active":"2026-02-22T21:45:01.133900+0000","last_became_peered":"2026-02-22T21:45:01.133900+0000","last_unstale":"2026-02-22T21:45:06.267514+0000","last_undegraded":"2026-02-22T21:45:06.267514+0000","last_fullsized":"2026-02-22T21:45:06.267514+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_clean_scrub_stamp":"2026-02-22T21:44:59.253183+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:32:00.074720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705667,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27092,"kb_used_data":128,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344748,"statfs":{"total":96636764160,"available":96609021952,"internally_reserved":0,"allocated":131072,"data_stored":37509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:45:09.874 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T21:45:09.874 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-22T21:45:09.874 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-22T21:45:09.875 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-22T21:45:10.048 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:10.048 INFO:teuthology.orchestra.run.trial048.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T21:45:10.056 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-22T21:45:10.056 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T21:45:10.071 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-22T21:45:10.071 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T21:45:10.071 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-workunit-policy-none}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64096', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror.sh workunit to test the rbd-mirror daemon in journal mode'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial048.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJJMis5tQRplKOa2XUKywCRJIwG5KxOlyoxvtJs22BJAkAwOpZt/4EmbZE9DgzvNfdZLt0tfEFyJGaAph1qzPS0=', 'trial104.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMrJCFPlCkMRhmAELVBPerDyM3cOH9yMWaJ5BdKzAXf9e5uCF8/sM2a8qcWXicwuezEoql2WVcFQ7Alnj+tRo3c='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror.sh']}, 'env': {'CEPH_ARGS': '', 'RBD_MIRROR_INSTANCES': '4', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T21:45:10.072 DEBUG:teuthology.orchestra.run.trial048:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T21:45:10.075 DEBUG:teuthology.orchestra.run.trial104:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T21:45:10.092 DEBUG:teuthology.orchestra.run.trial048:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:45:10.131 DEBUG:teuthology.orchestra.run.trial104:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T21:45:10.158 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:10.159 DEBUG:teuthology.orchestra.run.trial104:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T21:45:10.214 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T21:45:10.214 DEBUG:teuthology.orchestra.run.trial104:> stat /dev/vg_nvme/lv_1 2026-02-22T21:45:10.272 INFO:teuthology.orchestra.run.trial104.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T21:45:10.272 INFO:teuthology.orchestra.run.trial104.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:45:10.272 INFO:teuthology.orchestra.run.trial104.stdout:Device: 6h/6d Inode: 1292 Links: 1 2026-02-22T21:45:10.272 INFO:teuthology.orchestra.run.trial104.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:45:10.272 INFO:teuthology.orchestra.run.trial104.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:45:10.273 INFO:teuthology.orchestra.run.trial104.stdout:Access: 2026-02-22 21:44:03.979217878 +0000 2026-02-22T21:45:10.273 INFO:teuthology.orchestra.run.trial104.stdout:Modify: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.273 INFO:teuthology.orchestra.run.trial104.stdout:Change: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.273 INFO:teuthology.orchestra.run.trial104.stdout: Birth: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.273 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T21:45:10.335 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records in 2026-02-22T21:45:10.335 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records out 2026-02-22T21:45:10.335 INFO:teuthology.orchestra.run.trial104.stderr:512 bytes copied, 0.00018196 s, 2.8 MB/s 2026-02-22T21:45:10.336 DEBUG:teuthology.orchestra.run.trial104:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T21:45:10.395 DEBUG:teuthology.orchestra.run.trial104:> stat /dev/vg_nvme/lv_2 2026-02-22T21:45:10.453 INFO:teuthology.orchestra.run.trial104.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T21:45:10.453 INFO:teuthology.orchestra.run.trial104.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:45:10.453 INFO:teuthology.orchestra.run.trial104.stdout:Device: 6h/6d Inode: 1296 Links: 1 2026-02-22T21:45:10.454 INFO:teuthology.orchestra.run.trial104.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:45:10.454 INFO:teuthology.orchestra.run.trial104.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:45:10.454 INFO:teuthology.orchestra.run.trial104.stdout:Access: 2026-02-22 21:44:03.980217840 +0000 2026-02-22T21:45:10.454 INFO:teuthology.orchestra.run.trial104.stdout:Modify: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.454 INFO:teuthology.orchestra.run.trial104.stdout:Change: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.454 INFO:teuthology.orchestra.run.trial104.stdout: Birth: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.454 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T21:45:10.517 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records in 2026-02-22T21:45:10.518 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records out 2026-02-22T21:45:10.518 INFO:teuthology.orchestra.run.trial104.stderr:512 bytes copied, 0.000163546 s, 3.1 MB/s 2026-02-22T21:45:10.519 DEBUG:teuthology.orchestra.run.trial104:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T21:45:10.578 DEBUG:teuthology.orchestra.run.trial104:> stat /dev/vg_nvme/lv_3 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout:Device: 6h/6d Inode: 1303 Links: 1 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout:Access: 2026-02-22 21:44:03.980217840 +0000 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout:Modify: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout:Change: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.634 INFO:teuthology.orchestra.run.trial104.stdout: Birth: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.634 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T21:45:10.698 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records in 2026-02-22T21:45:10.698 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records out 2026-02-22T21:45:10.698 INFO:teuthology.orchestra.run.trial104.stderr:512 bytes copied, 0.000141415 s, 3.6 MB/s 2026-02-22T21:45:10.700 DEBUG:teuthology.orchestra.run.trial104:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T21:45:10.759 DEBUG:teuthology.orchestra.run.trial104:> stat /dev/vg_nvme/lv_4 2026-02-22T21:45:10.815 INFO:teuthology.orchestra.run.trial104.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T21:45:10.815 INFO:teuthology.orchestra.run.trial104.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T21:45:10.815 INFO:teuthology.orchestra.run.trial104.stdout:Device: 6h/6d Inode: 1301 Links: 1 2026-02-22T21:45:10.815 INFO:teuthology.orchestra.run.trial104.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T21:45:10.815 INFO:teuthology.orchestra.run.trial104.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T21:45:10.815 INFO:teuthology.orchestra.run.trial104.stdout:Access: 2026-02-22 21:44:03.980217840 +0000 2026-02-22T21:45:10.815 INFO:teuthology.orchestra.run.trial104.stdout:Modify: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.816 INFO:teuthology.orchestra.run.trial104.stdout:Change: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.816 INFO:teuthology.orchestra.run.trial104.stdout: Birth: 2026-02-22 21:43:23.077805620 +0000 2026-02-22T21:45:10.816 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T21:45:10.878 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records in 2026-02-22T21:45:10.878 INFO:teuthology.orchestra.run.trial104.stderr:1+0 records out 2026-02-22T21:45:10.878 INFO:teuthology.orchestra.run.trial104.stderr:512 bytes copied, 0.000166441 s, 3.1 MB/s 2026-02-22T21:45:10.879 DEBUG:teuthology.orchestra.run.trial104:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T21:45:10.938 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:45:10.938 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial104.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:45:10.938 INFO:tasks.ceph:Generating config... 2026-02-22T21:45:10.940 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client] rbd mirror image policy type = none 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:10.941 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:10.942 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:10.942 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T21:45:10.942 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T21:45:10.942 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T21:45:10.943 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T21:45:10.943 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T21:45:10.943 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T21:45:10.943 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T21:45:10.943 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T21:45:10.943 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T21:45:10.943 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T21:45:10.943 INFO:tasks.ceph:[osd] bluestore allocator = bitmap 2026-02-22T21:45:10.943 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T21:45:10.943 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T21:45:10.943 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T21:45:10.943 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T21:45:10.944 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T21:45:10.944 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-22T21:45:10.945 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-22T21:45:11.011 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.keyring 2026-02-22T21:45:11.014 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-22T21:45:11.089 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T21:45:11.154 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.104')] 2026-02-22T21:45:11.155 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.104', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125, 'rbd mirror image policy type': 'none'}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T21:45:11.156 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:11.156 DEBUG:teuthology.orchestra.run.trial104:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:45:11.212 DEBUG:teuthology.orchestra.run.trial104:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.104 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:11.283 INFO:teuthology.orchestra.run.trial104.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:11.283 INFO:teuthology.orchestra.run.trial104.stdout:monmaptool: generated fsid e1c6abd2-8277-4fd0-b513-f90ef487dc18 2026-02-22T21:45:11.283 INFO:teuthology.orchestra.run.trial104.stdout:setting min_mon_release = quincy 2026-02-22T21:45:11.283 INFO:teuthology.orchestra.run.trial104.stdout:epoch 0 2026-02-22T21:45:11.283 INFO:teuthology.orchestra.run.trial104.stdout:fsid e1c6abd2-8277-4fd0-b513-f90ef487dc18 2026-02-22T21:45:11.284 INFO:teuthology.orchestra.run.trial104.stdout:last_changed 2026-02-22T21:45:11.283157+0000 2026-02-22T21:45:11.284 INFO:teuthology.orchestra.run.trial104.stdout:created 2026-02-22T21:45:11.283157+0000 2026-02-22T21:45:11.284 INFO:teuthology.orchestra.run.trial104.stdout:min_mon_release 17 (quincy) 2026-02-22T21:45:11.284 INFO:teuthology.orchestra.run.trial104.stdout:election_strategy: 1 2026-02-22T21:45:11.284 INFO:teuthology.orchestra.run.trial104.stdout:0: [v2:10.20.193.104:3300/0,v1:10.20.193.104:6789/0] mon.a 2026-02-22T21:45:11.284 INFO:teuthology.orchestra.run.trial104.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-22T21:45:11.285 DEBUG:teuthology.orchestra.run.trial104:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T21:45:11.340 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID e1c6abd2-8277-4fd0-b513-f90ef487dc18... 2026-02-22T21:45:11.342 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T21:45:11.344 DEBUG:teuthology.orchestra.run.trial104:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout:[global] 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: chdir = "" 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: auth supported = cephx 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: filestore xattr use omap = true 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: mon clock drift allowed = 1.000 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: osd crush chooseleaf type = 0 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: auth debug = true 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.383 INFO:teuthology.orchestra.run.trial048.stdout: ms die on old message = true 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: ms die on bug = true 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon pg warn max object skew = 0 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd pool default size = 2 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon osd allow primary affinity = true 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon osd allow pg remap = true 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on no sortbitwise = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on too few osds = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon_allow_pool_size_one = true 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd default data pool replay window = 5 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon allow pool delete = true 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon cluster log file level = debug 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: debug asserts on shutdown = true 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon health detail to clog = false 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon host = 10.20.193.104 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon client directed command retry = 5 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: mon warn on pool no app = False 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: ms inject socket failures = 5000 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: fsid = e1c6abd2-8277-4fd0-b513-f90ef487dc18 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout:[osd] 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd journal size = 100 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd scrub load threshold = 5.0 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd scrub max interval = 600 2026-02-22T21:45:11.384 INFO:teuthology.orchestra.run.trial048.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd mclock skip benchmark = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd recover clone overlap = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd recovery max chunk = 1048576 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd debug shutdown = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd debug op order = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd debug verify stray on activate = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd debug trim objects = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd open classes on start = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd debug pg log writeout = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd map max advance = 10 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: journal zero on create = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: filestore apply finisher threads = 3 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: bdev debug aio = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd debug misdirected ops = true 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: bdev async discard = True 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: bdev enable discard = True 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: bluestore allocator = bitmap 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: bluestore block size = 96636764160 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: bluestore fsck on mount = True 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug bluefs = 1/20 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug bluestore = 1/20 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug ms = 1 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug osd = 20 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug rocksdb = 4/10 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: mon osd full ratio = 0.9 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: osd objectstore = bluestore 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout:[mgr] 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug ms = 1 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug mgr = 20 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug mon = 20 2026-02-22T21:45:11.385 INFO:teuthology.orchestra.run.trial048.stdout: debug auth = 20 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mgr/telemetry/nag = false 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout:[mon] 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: debug ms = 1 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: debug mon = 20 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: debug paxos = 20 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: debug auth = 20 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon data avail warn = 5 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon mgr mkfs grace = 240 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon osd reporter subtree level = osd 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon osd prime pg temp = true 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: # 1m isn't quite enough 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout:[client] 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: rgw cache enabled = true 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: rgw enable ops log = true 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: rgw enable usage log = true 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: debug rbd = 20 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: debug rbd_mirror = 15 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: log to stderr = False 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: rbd default features = 125 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: rbd mirror image policy type = none 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout:[mon.a] 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.0] 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.386 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.1] 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.2] 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.3] 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.4] 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.5] 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout:[client.mirror.6] 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.387 INFO:teuthology.orchestra.run.trial048.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.419 INFO:teuthology.orchestra.run.trial104.stdout:[global] 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: chdir = "" 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: auth supported = cephx 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: filestore xattr use omap = true 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: mon clock drift allowed = 1.000 2026-02-22T21:45:11.420 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: osd crush chooseleaf type = 0 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: auth debug = true 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: ms die on old message = true 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: ms die on bug = true 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: mon pg warn max object skew = 0 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T21:45:11.421 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: osd pool default size = 2 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon osd allow primary affinity = true 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon osd allow pg remap = true 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on legacy crush tunables = false 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on crush straw calc version zero = false 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on no sortbitwise = false 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on osd down out interval zero = false 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on too few osds = false 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T21:45:11.422 INFO:teuthology.orchestra.run.trial104.stdout: mon_allow_pool_size_one = true 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: osd default data pool replay window = 5 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: mon allow pool delete = true 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: mon cluster log file level = debug 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: debug asserts on shutdown = true 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: mon health detail to clog = false 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: mon host = 10.20.193.104 2026-02-22T21:45:11.423 INFO:teuthology.orchestra.run.trial104.stdout: mon client directed command retry = 5 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: mon warn on pool no app = False 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: ms inject socket failures = 5000 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: fsid = e1c6abd2-8277-4fd0-b513-f90ef487dc18 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout:[osd] 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: osd journal size = 100 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: osd scrub load threshold = 5.0 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: osd scrub max interval = 600 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: osd mclock profile = high_recovery_ops 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: osd mclock skip benchmark = true 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.424 INFO:teuthology.orchestra.run.trial104.stdout: osd recover clone overlap = true 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd recovery max chunk = 1048576 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd debug shutdown = true 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd debug op order = true 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd debug verify stray on activate = true 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd debug trim objects = true 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd open classes on start = true 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd debug pg log writeout = true 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: osd deep scrub update digest min age = 30 2026-02-22T21:45:11.425 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: osd map max advance = 10 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: journal zero on create = true 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: filestore ondisk finisher threads = 3 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: filestore apply finisher threads = 3 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: bdev debug aio = true 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: osd debug misdirected ops = true 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: bdev async discard = True 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: bdev enable discard = True 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: bluestore allocator = bitmap 2026-02-22T21:45:11.426 INFO:teuthology.orchestra.run.trial104.stdout: bluestore block size = 96636764160 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: bluestore fsck on mount = True 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: debug bluefs = 1/20 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: debug bluestore = 1/20 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: debug ms = 1 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: debug osd = 20 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: debug rocksdb = 4/10 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: mon osd full ratio = 0.9 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: mon osd nearfull ratio = 0.8 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: osd failsafe full ratio = 0.95 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: osd objectstore = bluestore 2026-02-22T21:45:11.427 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout:[mgr] 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: debug ms = 1 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: debug mgr = 20 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: debug mon = 20 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: debug auth = 20 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: mgr/telemetry/nag = false 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout:[mon] 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: debug ms = 1 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: debug mon = 20 2026-02-22T21:45:11.428 INFO:teuthology.orchestra.run.trial104.stdout: debug paxos = 20 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: debug auth = 20 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: mon data avail warn = 5 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: mon mgr mkfs grace = 240 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min pgs per osd = 4 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: mon osd reporter subtree level = osd 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: mon osd prime pg temp = true 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: mon reweight min bytes per osd = 10 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T21:45:11.429 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: # 1m isn't quite enough 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: mon_down_mkfs_grace = 2m 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: mon_warn_on_filestore_osds = false 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout:[client] 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: rgw cache enabled = true 2026-02-22T21:45:11.430 INFO:teuthology.orchestra.run.trial104.stdout: rgw enable ops log = true 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: rgw enable usage log = true 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: debug rbd = 20 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: debug rbd_mirror = 15 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: log to stderr = False 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: rbd default features = 125 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: rbd mirror image policy type = none 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout:[mon.a] 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.0] 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.431 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.1] 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.2] 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.3] 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.4] 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.5] 2026-02-22T21:45:11.432 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.433 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.433 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.6] 2026-02-22T21:45:11.433 INFO:teuthology.orchestra.run.trial104.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T21:45:11.433 INFO:teuthology.orchestra.run.trial104.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T21:45:11.434 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-22T21:45:11.434 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-22T21:45:11.517 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T21:45:11.517 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:11.518 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-22T21:45:11.575 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:11.575 DEBUG:teuthology.orchestra.run.trial104:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-22T21:45:11.632 INFO:tasks.ceph:Sending monmap to node ubuntu@trial048.front.sepia.ceph.com 2026-02-22T21:45:11.632 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:45:11.632 DEBUG:teuthology.orchestra.run.trial048:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T21:45:11.632 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T21:45:11.665 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:45:11.666 DEBUG:teuthology.orchestra.run.trial048:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:11.722 INFO:tasks.ceph:Sending monmap to node ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:45:11.722 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:11.723 DEBUG:teuthology.orchestra.run.trial104:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T21:45:11.723 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T21:45:11.759 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:11.759 DEBUG:teuthology.orchestra.run.trial104:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:11.814 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T21:45:11.815 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T21:45:11.815 DEBUG:teuthology.orchestra.run.trial048:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T21:45:11.858 INFO:teuthology.orchestra.run.trial048.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T21:45:11.860 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T21:45:11.861 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T21:45:11.861 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-22T21:45:11.895 INFO:teuthology.orchestra.run.trial048.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-22T21:45:11.905 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T21:45:11.939 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T21:45:11.951 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T21:45:12.025 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T21:45:12.036 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T21:45:12.109 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T21:45:12.121 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T21:45:12.195 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T21:45:12.205 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T21:45:12.280 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T21:45:12.291 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T21:45:12.365 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T21:45:12.376 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T21:45:12.449 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T21:45:12.460 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T21:45:12.533 INFO:teuthology.orchestra.run.trial104.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T21:45:12.544 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T21:45:12.544 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial048.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial104.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T21:45:12.544 DEBUG:teuthology.orchestra.run.trial104:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-22T21:45:12.608 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:45:12.608 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-22T21:45:12.608 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:45:12.608 DEBUG:teuthology.orchestra.run.trial104:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:45:12.675 INFO:teuthology.orchestra.run.trial104.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:45:13.248 INFO:teuthology.orchestra.run.trial104.stdout:Discarding blocks...Done. 2026-02-22T21:45:13.250 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial104.front.sepia.ceph.com -o noatime 2026-02-22T21:45:13.250 DEBUG:teuthology.orchestra.run.trial104:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-22T21:45:13.319 DEBUG:teuthology.orchestra.run.trial104:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-22T21:45:13.385 DEBUG:teuthology.orchestra.run.trial104:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-22T21:45:13.450 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:45:13.451 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-22T21:45:13.451 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:45:13.451 DEBUG:teuthology.orchestra.run.trial104:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T21:45:13.515 INFO:teuthology.orchestra.run.trial104.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:45:13.516 INFO:teuthology.orchestra.run.trial104.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:45:13.517 INFO:teuthology.orchestra.run.trial104.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:45:14.074 INFO:teuthology.orchestra.run.trial104.stdout:Discarding blocks...Done. 2026-02-22T21:45:14.075 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial104.front.sepia.ceph.com -o noatime 2026-02-22T21:45:14.075 DEBUG:teuthology.orchestra.run.trial104:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-22T21:45:14.143 DEBUG:teuthology.orchestra.run.trial104:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-22T21:45:14.208 DEBUG:teuthology.orchestra.run.trial104:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-22T21:45:14.272 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T21:45:14.273 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-22T21:45:14.273 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:45:14.273 DEBUG:teuthology.orchestra.run.trial104:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T21:45:14.337 INFO:teuthology.orchestra.run.trial104.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T21:45:14.337 INFO:teuthology.orchestra.run.trial104.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T21:45:14.337 INFO:teuthology.orchestra.run.trial104.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T21:45:14.337 INFO:teuthology.orchestra.run.trial104.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T21:45:14.338 INFO:teuthology.orchestra.run.trial104.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T21:45:14.338 INFO:teuthology.orchestra.run.trial104.stdout: = sunit=0 swidth=0 blks 2026-02-22T21:45:14.338 INFO:teuthology.orchestra.run.trial104.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T21:45:14.338 INFO:teuthology.orchestra.run.trial104.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T21:45:14.338 INFO:teuthology.orchestra.run.trial104.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T21:45:14.338 INFO:teuthology.orchestra.run.trial104.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T21:45:14.894 INFO:teuthology.orchestra.run.trial104.stdout:Discarding blocks...Done. 2026-02-22T21:45:14.896 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial104.front.sepia.ceph.com -o noatime 2026-02-22T21:45:14.896 DEBUG:teuthology.orchestra.run.trial104:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-22T21:45:14.963 DEBUG:teuthology.orchestra.run.trial104:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-22T21:45:15.029 DEBUG:teuthology.orchestra.run.trial104:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:15.107 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:15.105+0000 7fb28100d780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-22T21:45:15.107 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:15.106+0000 7fb28100d780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-22T21:45:15.107 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:15.106+0000 7fb28100d780 -1 bdev(0x55668ca6f800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-22T21:45:15.107 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:15.106+0000 7fb28100d780 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-22T21:45:19.000 DEBUG:teuthology.orchestra.run.trial104:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-22T21:45:19.065 DEBUG:teuthology.orchestra.run.trial104:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:19.144 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:19.142+0000 7fe047e0c780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-22T21:45:19.144 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:19.143+0000 7fe047e0c780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-22T21:45:19.144 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:19.143+0000 7fe047e0c780 -1 bdev(0x561d7773d800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-22T21:45:19.144 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:19.143+0000 7fe047e0c780 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-22T21:45:23.036 DEBUG:teuthology.orchestra.run.trial104:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-22T21:45:23.102 DEBUG:teuthology.orchestra.run.trial104:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:23.180 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:23.178+0000 7ff914c2f780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-22T21:45:23.181 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:23.178+0000 7ff914c2f780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-22T21:45:23.181 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:23.178+0000 7ff914c2f780 -1 bdev(0x55b48af7d800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-22T21:45:23.181 INFO:teuthology.orchestra.run.trial104.stderr:2026-02-22T21:45:23.178+0000 7ff914c2f780 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-22T21:45:27.073 DEBUG:teuthology.orchestra.run.trial104:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-22T21:45:27.137 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T21:45:27.137 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:45:27.137 DEBUG:teuthology.orchestra.run.trial048:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-22T21:45:27.164 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.164 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-22T21:45:27.201 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.202 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-22T21:45:27.266 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.267 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-22T21:45:27.329 DEBUG:teuthology.orchestra.run.trial048:> set -ex 2026-02-22T21:45:27.329 DEBUG:teuthology.orchestra.run.trial048:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-22T21:45:27.345 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.345 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-22T21:45:27.385 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.385 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-22T21:45:27.442 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.442 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-22T21:45:27.498 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.498 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-22T21:45:27.554 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.554 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-22T21:45:27.610 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.610 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-22T21:45:27.666 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.666 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-22T21:45:27.724 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:27.724 DEBUG:teuthology.orchestra.run.trial104:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-22T21:45:27.784 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T21:45:27.784 DEBUG:teuthology.orchestra.run.trial104:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-22T21:45:27.853 INFO:teuthology.orchestra.run.trial104.stdout:[mgr.x] 2026-02-22T21:45:27.854 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDneJtpOPMkMxAApk1drAscZ5U4znkhzTP4jw== 2026-02-22T21:45:27.854 INFO:teuthology.orchestra.run.trial104.stdout:[osd.0] 2026-02-22T21:45:27.854 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDreJtp8ExiBhAAU/u4mkHAGsNYF3xmqVlUvA== 2026-02-22T21:45:27.854 INFO:teuthology.orchestra.run.trial104.stdout:[osd.1] 2026-02-22T21:45:27.854 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDveJtpA2KXCBAAc2YdDcImEGjnkUE8Urvc1A== 2026-02-22T21:45:27.854 INFO:teuthology.orchestra.run.trial104.stdout:[osd.2] 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDzeJtpGcivChAAqjVdXWadkHCmYyrOoA4+uw== 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout:[client.0] 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDneJtp1SlSNRAA8VO0Z5nqBM9ITDDIO/0S7g== 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror] 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDneJtpeLADOBAAPtEhZFEAg1PIjt+UTAhvMA== 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.0] 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpfHmHARAA5M/6tT4KKafpgp0wGxP3Uw== 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.1] 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpm2GABhAAHPZZzfqwd+VsapU78ruh7A== 2026-02-22T21:45:27.855 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.2] 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpkKeeCxAAz0oKYW0N9esgxx0zkJAjGA== 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.3] 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtp8AytEBAAPIB+q093oFq93fwrgEepcA== 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.4] 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpkTjNFRAAN2z+1cTQZHLdEKG8ENg9DQ== 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.5] 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpvn3EGhAAJPSXHG0soFAG2SEcqdEObg== 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.6] 2026-02-22T21:45:27.856 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtp1kHEHxAABJyjyHqrGVzimPqJB9t/4g== 2026-02-22T21:45:27.858 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T21:45:27.939 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:45:28.017 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:45:28.098 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T21:45:28.178 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.258 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.338 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.421 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.500 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.578 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.660 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.740 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.820 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T21:45:28.901 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T21:45:28.901 DEBUG:teuthology.orchestra.run.trial104:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-22T21:45:28.965 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-22T21:45:29.051 DEBUG:teuthology.orchestra.run.trial104:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-22T21:45:29.076 DEBUG:teuthology.orchestra.run.trial104:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T21:45:29.132 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-22T21:45:29.132 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-22T21:45:29.132 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-22T21:45:29.174 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-22T21:45:29.174 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-22T21:45:29.174 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-22T21:45:29.174 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-22T21:45:29.176 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-22T21:45:29.176 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T21:45:29.176 DEBUG:teuthology.orchestra.run.trial104:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-22T21:45:29.273 INFO:teuthology.orchestra.run.trial104.stderr:adjusted tunables profile to default 2026-02-22T21:45:29.281 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T21:45:29.281 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-22T21:45:29.281 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:29.281 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-22T21:45:29.305 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:29.305 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-22T21:45:29.369 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:45:29.369 DEBUG:teuthology.orchestra.run.trial104:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-22T21:45:29.431 DEBUG:teuthology.orchestra.run.trial104:> sudo ceph --cluster cluster2 osd new 41ca61ad-dac2-4161-b228-821056447285 0 2026-02-22T21:45:29.504 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.503+0000 7fc4adf33140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T21:45:29.573 INFO:teuthology.orchestra.run.trial104.stdout:0 2026-02-22T21:45:29.580 DEBUG:teuthology.orchestra.run.trial104:> sudo ceph --cluster cluster2 osd new 3f8ea715-66fc-455f-b443-a336ef783866 1 2026-02-22T21:45:29.621 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.620+0000 7fc4adf33140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T21:45:29.643 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.642+0000 7fc4adf33140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T21:45:29.669 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.667+0000 7fc4adf33140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T21:45:29.683 INFO:teuthology.orchestra.run.trial104.stdout:1 2026-02-22T21:45:29.690 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.688+0000 7fc4adf33140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T21:45:29.691 DEBUG:teuthology.orchestra.run.trial104:> sudo ceph --cluster cluster2 osd new ee29ada6-f1b6-41d9-aa45-93db0f1af5c4 2 2026-02-22T21:45:29.715 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.713+0000 7fc4adf33140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T21:45:29.766 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.764+0000 7fc4adf33140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T21:45:29.794 INFO:teuthology.orchestra.run.trial104.stdout:2 2026-02-22T21:45:29.801 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-22T21:45:29.802 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-22T21:45:29.804 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-22T21:45:29.804 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-22T21:45:29.804 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-22T21:45:29.806 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-22T21:45:29.806 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-22T21:45:29.807 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-22T21:45:29.809 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-22T21:45:29.809 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:45:29.825 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T21:45:29.825 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T21:45:29.825 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr: from numpy import show_config as show_numpy_config 2026-02-22T21:45:29.827 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.826+0000 7fc4adf33140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T21:45:29.852 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.850+0000 7fc4adf33140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T21:45:29.906 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.904+0000 7fc4adf33140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T21:45:29.909 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:29.909 INFO:teuthology.orchestra.run.trial104.stdout:{"epoch":5,"fsid":"e1c6abd2-8277-4fd0-b513-f90ef487dc18","created":"2026-02-22T21:45:29.235879+0000","modified":"2026-02-22T21:45:29.793510+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:45:29.793510+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"41ca61ad-dac2-4161-b228-821056447285","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"3f8ea715-66fc-455f-b443-a336ef783866","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"ee29ada6-f1b6-41d9-aa45-93db0f1af5c4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:29.917 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-22T21:45:29.917 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T21:45:29.946 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.945+0000 7fc4adf33140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T21:45:29.970 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.969+0000 7fc4adf33140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T21:45:29.994 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:29.992+0000 7fc4adf33140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T21:45:30.018 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.016+0000 7fc4adf33140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T21:45:30.038 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.037+0000 7fc4adf33140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T21:45:30.191 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.190+0000 7fc4adf33140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T21:45:30.219 DEBUG:teuthology.orchestra.run.trial104:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:45:30.246 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.245+0000 7fc4adf33140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T21:45:30.267 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.266+0000 7fc4adf33140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T21:45:30.289 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.287+0000 7fc4adf33140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T21:45:30.312 INFO:teuthology.misc.health.trial104.stdout: 2026-02-22T21:45:30.312 INFO:teuthology.misc.health.trial104.stdout:{"epoch":5,"fsid":"e1c6abd2-8277-4fd0-b513-f90ef487dc18","created":"2026-02-22T21:45:29.235879+0000","modified":"2026-02-22T21:45:29.793510+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:45:29.793510+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"41ca61ad-dac2-4161-b228-821056447285","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"3f8ea715-66fc-455f-b443-a336ef783866","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"ee29ada6-f1b6-41d9-aa45-93db0f1af5c4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:30.317 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:45:30.341 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.340+0000 7fc4adf33140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:45:30.361 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.360+0000 7fc4adf33140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T21:45:30.413 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.412+0000 7fc4adf33140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T21:45:30.735 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.733+0000 7fc4adf33140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T21:45:30.756 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:30.754+0000 7fc4adf33140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T21:45:31.086 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:31.085+0000 7fc4adf33140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T21:45:31.463 INFO:tasks.ceph.cluster2.osd.1.trial104.stderr:2026-02-22T21:45:31.461+0000 7f56be716780 -1 Falling back to public interface 2026-02-22T21:45:31.713 INFO:tasks.ceph.cluster2.osd.0.trial104.stderr:2026-02-22T21:45:31.711+0000 7faf747b4780 -1 Falling back to public interface 2026-02-22T21:45:31.718 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T21:45:31.716+0000 7f46c4e27780 -1 Falling back to public interface 2026-02-22T21:45:32.243 INFO:tasks.ceph.cluster2.mgr.x.trial048.stderr:2026-02-22T21:45:32.242+0000 7fc3cb66b640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T21:45:35.980 INFO:tasks.ceph.cluster2.osd.1.trial104.stderr:2026-02-22T21:45:35.979+0000 7f56be716780 -1 osd.1 0 log_to_monitors true 2026-02-22T21:45:36.179 INFO:tasks.ceph.cluster2.osd.0.trial104.stderr:2026-02-22T21:45:36.178+0000 7faf747b4780 -1 osd.0 0 log_to_monitors true 2026-02-22T21:45:36.211 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T21:45:36.210+0000 7f46c4e27780 -1 osd.2 0 log_to_monitors true 2026-02-22T21:45:36.620 DEBUG:teuthology.orchestra.run.trial104:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:45:36.774 INFO:teuthology.misc.health.trial104.stdout: 2026-02-22T21:45:36.775 INFO:teuthology.misc.health.trial104.stdout:{"epoch":6,"fsid":"e1c6abd2-8277-4fd0-b513-f90ef487dc18","created":"2026-02-22T21:45:29.235879+0000","modified":"2026-02-22T21:45:36.032222+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T21:45:29.793510+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"41ca61ad-dac2-4161-b228-821056447285","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"3f8ea715-66fc-455f-b443-a336ef783866","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"ee29ada6-f1b6-41d9-aa45-93db0f1af5c4","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:36.783 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T21:45:37.046 INFO:tasks.ceph.cluster2.osd.1.trial104.stderr:2026-02-22T21:45:37.046+0000 7f56ba681640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T21:45:37.048 INFO:tasks.ceph.cluster2.osd.1.trial104.stderr:2026-02-22T21:45:37.048+0000 7f56b24b4640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:45:38.047 INFO:tasks.ceph.cluster2.osd.0.trial104.stderr:2026-02-22T21:45:38.047+0000 7faf7071f640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T21:45:38.047 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T21:45:38.047+0000 7f46c0d92640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T21:45:38.049 INFO:tasks.ceph.cluster2.osd.0.trial104.stderr:2026-02-22T21:45:38.049+0000 7faf68552640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:45:38.049 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T21:45:38.049+0000 7f46b8bc5640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T21:45:43.086 DEBUG:teuthology.orchestra.run.trial104:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:45:43.237 INFO:teuthology.misc.health.trial104.stdout: 2026-02-22T21:45:43.238 INFO:teuthology.misc.health.trial104.stdout:{"epoch":12,"fsid":"e1c6abd2-8277-4fd0-b513-f90ef487dc18","created":"2026-02-22T21:45:29.235879+0000","modified":"2026-02-22T21:45:42.043378+0000","last_up_change":"2026-02-22T21:45:39.040408+0000","last_in_change":"2026-02-22T21:45:29.793510+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:45:39.306938+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"41ca61ad-dac2-4161-b228-821056447285","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6808","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6809","nonce":895813024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6810","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6811","nonce":895813024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6814","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6815","nonce":895813024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6812","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6813","nonce":895813024}]},"public_addr":"10.20.193.104:6809/895813024","cluster_addr":"10.20.193.104:6811/895813024","heartbeat_back_addr":"10.20.193.104:6815/895813024","heartbeat_front_addr":"10.20.193.104:6813/895813024","state":["exists","up"]},{"osd":1,"uuid":"3f8ea715-66fc-455f-b443-a336ef783866","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6800","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6801","nonce":251451549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6802","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6803","nonce":251451549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6806","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6807","nonce":251451549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6804","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6805","nonce":251451549}]},"public_addr":"10.20.193.104:6801/251451549","cluster_addr":"10.20.193.104:6803/251451549","heartbeat_back_addr":"10.20.193.104:6807/251451549","heartbeat_front_addr":"10.20.193.104:6805/251451549","state":["exists","up"]},{"osd":2,"uuid":"ee29ada6-f1b6-41d9-aa45-93db0f1af5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6816","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6817","nonce":2738971318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6818","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6819","nonce":2738971318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6822","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6823","nonce":2738971318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6820","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6821","nonce":2738971318}]},"public_addr":"10.20.193.104:6817/2738971318","cluster_addr":"10.20.193.104:6819/2738971318","heartbeat_back_addr":"10.20.193.104:6823/2738971318","heartbeat_front_addr":"10.20.193.104:6821/2738971318","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.197574+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:36.954361+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.258932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:43.244 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T21:45:43.244 INFO:tasks.ceph:Creating RBD pool 2026-02-22T21:45:43.245 DEBUG:teuthology.orchestra.run.trial104:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-22T21:45:44.050 INFO:teuthology.orchestra.run.trial104.stderr:pool 'rbd' created 2026-02-22T21:45:44.062 DEBUG:teuthology.orchestra.run.trial104:> rbd --cluster cluster2 pool init rbd 2026-02-22T21:45:47.063 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-22T21:45:47.064 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-22T21:45:47.064 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-22T21:45:47.065 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-22T21:45:47.065 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T21:45:47.243 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:47.250 INFO:teuthology.orchestra.run.trial104.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6826","nonce":1285864800},{"type":"v1","addr":"10.20.193.48:6827","nonce":1285864800}]},"active_addr":"10.20.193.48:6827/1285864800","active_change":"2026-02-22T21:45:31.087522+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":3883412500}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":2258175345}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":1196764850}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.48:0","nonce":2682947521}]}]} 2026-02-22T21:45:47.258 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-22T21:45:47.258 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-22T21:45:47.258 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:45:47.414 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:47.414 INFO:teuthology.orchestra.run.trial104.stdout:{"epoch":16,"fsid":"e1c6abd2-8277-4fd0-b513-f90ef487dc18","created":"2026-02-22T21:45:29.235879+0000","modified":"2026-02-22T21:45:47.053381+0000","last_up_change":"2026-02-22T21:45:39.040408+0000","last_in_change":"2026-02-22T21:45:29.793510+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:45:39.306938+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:45:43.402955+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"41ca61ad-dac2-4161-b228-821056447285","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6808","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6809","nonce":895813024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6810","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6811","nonce":895813024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6814","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6815","nonce":895813024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6812","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6813","nonce":895813024}]},"public_addr":"10.20.193.104:6809/895813024","cluster_addr":"10.20.193.104:6811/895813024","heartbeat_back_addr":"10.20.193.104:6815/895813024","heartbeat_front_addr":"10.20.193.104:6813/895813024","state":["exists","up"]},{"osd":1,"uuid":"3f8ea715-66fc-455f-b443-a336ef783866","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6800","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6801","nonce":251451549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6802","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6803","nonce":251451549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6806","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6807","nonce":251451549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6804","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6805","nonce":251451549}]},"public_addr":"10.20.193.104:6801/251451549","cluster_addr":"10.20.193.104:6803/251451549","heartbeat_back_addr":"10.20.193.104:6807/251451549","heartbeat_front_addr":"10.20.193.104:6805/251451549","state":["exists","up"]},{"osd":2,"uuid":"ee29ada6-f1b6-41d9-aa45-93db0f1af5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6816","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6817","nonce":2738971318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6818","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6819","nonce":2738971318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6822","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6823","nonce":2738971318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6820","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6821","nonce":2738971318}]},"public_addr":"10.20.193.104:6817/2738971318","cluster_addr":"10.20.193.104:6819/2738971318","heartbeat_back_addr":"10.20.193.104:6823/2738971318","heartbeat_front_addr":"10.20.193.104:6821/2738971318","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.197574+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:36.954361+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.258932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:47.423 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-22T21:45:47.423 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T21:45:47.579 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:47.579 INFO:teuthology.orchestra.run.trial104.stdout:{"epoch":16,"fsid":"e1c6abd2-8277-4fd0-b513-f90ef487dc18","created":"2026-02-22T21:45:29.235879+0000","modified":"2026-02-22T21:45:47.053381+0000","last_up_change":"2026-02-22T21:45:39.040408+0000","last_in_change":"2026-02-22T21:45:29.793510+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:45:39.306938+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:45:43.402955+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"41ca61ad-dac2-4161-b228-821056447285","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6808","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6809","nonce":895813024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6810","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6811","nonce":895813024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6814","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6815","nonce":895813024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6812","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6813","nonce":895813024}]},"public_addr":"10.20.193.104:6809/895813024","cluster_addr":"10.20.193.104:6811/895813024","heartbeat_back_addr":"10.20.193.104:6815/895813024","heartbeat_front_addr":"10.20.193.104:6813/895813024","state":["exists","up"]},{"osd":1,"uuid":"3f8ea715-66fc-455f-b443-a336ef783866","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6800","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6801","nonce":251451549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6802","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6803","nonce":251451549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6806","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6807","nonce":251451549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6804","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6805","nonce":251451549}]},"public_addr":"10.20.193.104:6801/251451549","cluster_addr":"10.20.193.104:6803/251451549","heartbeat_back_addr":"10.20.193.104:6807/251451549","heartbeat_front_addr":"10.20.193.104:6805/251451549","state":["exists","up"]},{"osd":2,"uuid":"ee29ada6-f1b6-41d9-aa45-93db0f1af5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6816","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6817","nonce":2738971318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6818","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6819","nonce":2738971318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6822","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6823","nonce":2738971318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6820","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6821","nonce":2738971318}]},"public_addr":"10.20.193.104:6817/2738971318","cluster_addr":"10.20.193.104:6819/2738971318","heartbeat_back_addr":"10.20.193.104:6823/2738971318","heartbeat_front_addr":"10.20.193.104:6821/2738971318","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.197574+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:36.954361+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.258932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T21:45:47.587 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-22T21:45:47.588 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-22T21:45:47.588 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-22T21:45:47.675 INFO:teuthology.orchestra.run.trial104.stdout:38654705668 2026-02-22T21:45:47.676 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T21:45:47.677 INFO:teuthology.orchestra.run.trial104.stdout:34359738371 2026-02-22T21:45:47.677 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T21:45:47.678 INFO:teuthology.orchestra.run.trial104.stdout:38654705669 2026-02-22T21:45:47.679 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T21:45:47.875 INFO:teuthology.orchestra.run.trial104.stdout:38654705668 2026-02-22T21:45:47.877 INFO:teuthology.orchestra.run.trial104.stdout:34359738370 2026-02-22T21:45:47.880 INFO:teuthology.orchestra.run.trial104.stdout:38654705667 2026-02-22T21:45:47.884 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.0 2026-02-22T21:45:47.887 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.1 2026-02-22T21:45:47.889 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T21:45:48.885 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T21:45:48.887 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T21:45:48.890 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T21:45:49.044 INFO:teuthology.orchestra.run.trial104.stdout:38654705668 2026-02-22T21:45:49.047 INFO:teuthology.orchestra.run.trial104.stdout:34359738370 2026-02-22T21:45:49.054 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.0 2026-02-22T21:45:49.058 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738370 for osd.1 2026-02-22T21:45:49.091 INFO:teuthology.orchestra.run.trial104.stdout:38654705667 2026-02-22T21:45:49.099 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T21:45:50.054 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T21:45:50.060 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T21:45:50.100 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T21:45:50.214 INFO:teuthology.orchestra.run.trial104.stdout:38654705669 2026-02-22T21:45:50.225 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705669 for osd.0 2026-02-22T21:45:50.225 DEBUG:teuthology.parallel:result is None 2026-02-22T21:45:50.257 INFO:teuthology.orchestra.run.trial104.stdout:34359738372 2026-02-22T21:45:50.266 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738371 got 34359738372 for osd.1 2026-02-22T21:45:50.267 DEBUG:teuthology.parallel:result is None 2026-02-22T21:45:50.297 INFO:teuthology.orchestra.run.trial104.stdout:38654705668 2026-02-22T21:45:50.306 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705668 for osd.2 2026-02-22T21:45:50.307 DEBUG:teuthology.parallel:result is None 2026-02-22T21:45:50.307 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T21:45:50.307 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T21:45:50.500 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:50.500 INFO:teuthology.orchestra.run.trial104.stderr:dumped all 2026-02-22T21:45:50.508 INFO:teuthology.orchestra.run.trial104.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T21:45:49.092507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82456,"kb_used_data":1552,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033064,"statfs":{"total":289910292480,"available":289825857536,"internally_reserved":0,"allocated":1589248,"data_stored":1292312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":82767187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.042660"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055786+0000","last_change":"2026-02-22T21:45:47.055786+0000","last_active":"2026-02-22T21:45:47.055786+0000","last_peered":"2026-02-22T21:45:47.055786+0000","last_clean":"2026-02-22T21:45:47.055786+0000","last_became_active":"2026-02-22T21:45:45.053797+0000","last_became_peered":"2026-02-22T21:45:45.053797+0000","last_unstale":"2026-02-22T21:45:47.055786+0000","last_undegraded":"2026-02-22T21:45:47.055786+0000","last_fullsized":"2026-02-22T21:45:47.055786+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:53:17.577669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050143900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055663+0000","last_change":"2026-02-22T21:45:47.055663+0000","last_active":"2026-02-22T21:45:47.055663+0000","last_peered":"2026-02-22T21:45:47.055663+0000","last_clean":"2026-02-22T21:45:47.055663+0000","last_became_active":"2026-02-22T21:45:45.053799+0000","last_became_peered":"2026-02-22T21:45:45.053799+0000","last_unstale":"2026-02-22T21:45:47.055663+0000","last_undegraded":"2026-02-22T21:45:47.055663+0000","last_fullsized":"2026-02-22T21:45:47.055663+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:27:16.854273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0003992,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055898+0000","last_change":"2026-02-22T21:45:47.055898+0000","last_active":"2026-02-22T21:45:47.055898+0000","last_peered":"2026-02-22T21:45:47.055898+0000","last_clean":"2026-02-22T21:45:47.055898+0000","last_became_active":"2026-02-22T21:45:45.053506+0000","last_became_peered":"2026-02-22T21:45:45.053506+0000","last_unstale":"2026-02-22T21:45:47.055898+0000","last_undegraded":"2026-02-22T21:45:47.055898+0000","last_fullsized":"2026-02-22T21:45:47.055898+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:10:14.553071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060315800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055731+0000","last_change":"2026-02-22T21:45:47.055731+0000","last_active":"2026-02-22T21:45:47.055731+0000","last_peered":"2026-02-22T21:45:47.055731+0000","last_clean":"2026-02-22T21:45:47.055731+0000","last_became_active":"2026-02-22T21:45:45.053788+0000","last_became_peered":"2026-02-22T21:45:45.053788+0000","last_unstale":"2026-02-22T21:45:47.055731+0000","last_undegraded":"2026-02-22T21:45:47.055731+0000","last_fullsized":"2026-02-22T21:45:47.055731+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:15:31.306502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048115199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.057685+0000","last_change":"2026-02-22T21:45:47.057685+0000","last_active":"2026-02-22T21:45:47.057685+0000","last_peered":"2026-02-22T21:45:47.057685+0000","last_clean":"2026-02-22T21:45:47.057685+0000","last_became_active":"2026-02-22T21:45:45.053848+0000","last_became_peered":"2026-02-22T21:45:45.053848+0000","last_unstale":"2026-02-22T21:45:47.057685+0000","last_undegraded":"2026-02-22T21:45:47.057685+0000","last_fullsized":"2026-02-22T21:45:47.057685+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:30:42.167683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089195400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.106196+0000","last_change":"2026-02-22T21:45:47.106335+0000","last_active":"2026-02-22T21:45:47.106196+0000","last_peered":"2026-02-22T21:45:47.106196+0000","last_clean":"2026-02-22T21:45:47.106196+0000","last_became_active":"2026-02-22T21:45:45.053963+0000","last_became_peered":"2026-02-22T21:45:45.053963+0000","last_unstale":"2026-02-22T21:45:47.106196+0000","last_undegraded":"2026-02-22T21:45:47.106196+0000","last_fullsized":"2026-02-22T21:45:47.106196+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:40:42.243266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000224897,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.106182+0000","last_change":"2026-02-22T21:45:47.106341+0000","last_active":"2026-02-22T21:45:47.106182+0000","last_peered":"2026-02-22T21:45:47.106182+0000","last_clean":"2026-02-22T21:45:47.106182+0000","last_became_active":"2026-02-22T21:45:45.053834+0000","last_became_peered":"2026-02-22T21:45:45.053834+0000","last_unstale":"2026-02-22T21:45:47.106182+0000","last_undegraded":"2026-02-22T21:45:47.106182+0000","last_fullsized":"2026-02-22T21:45:47.106182+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:44:15.059855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000242379,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.056034+0000","last_change":"2026-02-22T21:45:47.056034+0000","last_active":"2026-02-22T21:45:47.056034+0000","last_peered":"2026-02-22T21:45:47.056034+0000","last_clean":"2026-02-22T21:45:47.056034+0000","last_became_active":"2026-02-22T21:45:45.053434+0000","last_became_peered":"2026-02-22T21:45:45.053434+0000","last_unstale":"2026-02-22T21:45:47.056034+0000","last_undegraded":"2026-02-22T21:45:47.056034+0000","last_fullsized":"2026-02-22T21:45:47.056034+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:45.436732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076235300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":109,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055093+0000","last_change":"2026-02-22T21:45:41.044949+0000","last_active":"2026-02-22T21:45:47.055093+0000","last_peered":"2026-02-22T21:45:47.055093+0000","last_clean":"2026-02-22T21:45:47.055093+0000","last_became_active":"2026-02-22T21:45:41.044711+0000","last_became_peered":"2026-02-22T21:45:41.044711+0000","last_unstale":"2026-02-22T21:45:47.055093+0000","last_undegraded":"2026-02-22T21:45:47.055093+0000","last_fullsized":"2026-02-22T21:45:47.055093+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_clean_scrub_stamp":"2026-02-22T21:45:40.040843+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:25:57.694568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":122880,"data_stored":33679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":729088,"data_stored":627531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:45:50.508 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T21:45:50.665 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:50.666 INFO:teuthology.orchestra.run.trial104.stderr:dumped all 2026-02-22T21:45:50.675 INFO:teuthology.orchestra.run.trial104.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T21:45:49.092507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":17,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82456,"kb_used_data":1552,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033064,"statfs":{"total":289910292480,"available":289825857536,"internally_reserved":0,"allocated":1589248,"data_stored":1292312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":82767187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.042660"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055786+0000","last_change":"2026-02-22T21:45:47.055786+0000","last_active":"2026-02-22T21:45:47.055786+0000","last_peered":"2026-02-22T21:45:47.055786+0000","last_clean":"2026-02-22T21:45:47.055786+0000","last_became_active":"2026-02-22T21:45:45.053797+0000","last_became_peered":"2026-02-22T21:45:45.053797+0000","last_unstale":"2026-02-22T21:45:47.055786+0000","last_undegraded":"2026-02-22T21:45:47.055786+0000","last_fullsized":"2026-02-22T21:45:47.055786+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:53:17.577669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050143900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055663+0000","last_change":"2026-02-22T21:45:47.055663+0000","last_active":"2026-02-22T21:45:47.055663+0000","last_peered":"2026-02-22T21:45:47.055663+0000","last_clean":"2026-02-22T21:45:47.055663+0000","last_became_active":"2026-02-22T21:45:45.053799+0000","last_became_peered":"2026-02-22T21:45:45.053799+0000","last_unstale":"2026-02-22T21:45:47.055663+0000","last_undegraded":"2026-02-22T21:45:47.055663+0000","last_fullsized":"2026-02-22T21:45:47.055663+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:27:16.854273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0003992,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055898+0000","last_change":"2026-02-22T21:45:47.055898+0000","last_active":"2026-02-22T21:45:47.055898+0000","last_peered":"2026-02-22T21:45:47.055898+0000","last_clean":"2026-02-22T21:45:47.055898+0000","last_became_active":"2026-02-22T21:45:45.053506+0000","last_became_peered":"2026-02-22T21:45:45.053506+0000","last_unstale":"2026-02-22T21:45:47.055898+0000","last_undegraded":"2026-02-22T21:45:47.055898+0000","last_fullsized":"2026-02-22T21:45:47.055898+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:10:14.553071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060315800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055731+0000","last_change":"2026-02-22T21:45:47.055731+0000","last_active":"2026-02-22T21:45:47.055731+0000","last_peered":"2026-02-22T21:45:47.055731+0000","last_clean":"2026-02-22T21:45:47.055731+0000","last_became_active":"2026-02-22T21:45:45.053788+0000","last_became_peered":"2026-02-22T21:45:45.053788+0000","last_unstale":"2026-02-22T21:45:47.055731+0000","last_undegraded":"2026-02-22T21:45:47.055731+0000","last_fullsized":"2026-02-22T21:45:47.055731+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:15:31.306502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048115199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.057685+0000","last_change":"2026-02-22T21:45:47.057685+0000","last_active":"2026-02-22T21:45:47.057685+0000","last_peered":"2026-02-22T21:45:47.057685+0000","last_clean":"2026-02-22T21:45:47.057685+0000","last_became_active":"2026-02-22T21:45:45.053848+0000","last_became_peered":"2026-02-22T21:45:45.053848+0000","last_unstale":"2026-02-22T21:45:47.057685+0000","last_undegraded":"2026-02-22T21:45:47.057685+0000","last_fullsized":"2026-02-22T21:45:47.057685+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:30:42.167683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089195400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.106196+0000","last_change":"2026-02-22T21:45:47.106335+0000","last_active":"2026-02-22T21:45:47.106196+0000","last_peered":"2026-02-22T21:45:47.106196+0000","last_clean":"2026-02-22T21:45:47.106196+0000","last_became_active":"2026-02-22T21:45:45.053963+0000","last_became_peered":"2026-02-22T21:45:45.053963+0000","last_unstale":"2026-02-22T21:45:47.106196+0000","last_undegraded":"2026-02-22T21:45:47.106196+0000","last_fullsized":"2026-02-22T21:45:47.106196+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:40:42.243266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000224897,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.106182+0000","last_change":"2026-02-22T21:45:47.106341+0000","last_active":"2026-02-22T21:45:47.106182+0000","last_peered":"2026-02-22T21:45:47.106182+0000","last_clean":"2026-02-22T21:45:47.106182+0000","last_became_active":"2026-02-22T21:45:45.053834+0000","last_became_peered":"2026-02-22T21:45:45.053834+0000","last_unstale":"2026-02-22T21:45:47.106182+0000","last_undegraded":"2026-02-22T21:45:47.106182+0000","last_fullsized":"2026-02-22T21:45:47.106182+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:44:15.059855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000242379,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.056034+0000","last_change":"2026-02-22T21:45:47.056034+0000","last_active":"2026-02-22T21:45:47.056034+0000","last_peered":"2026-02-22T21:45:47.056034+0000","last_clean":"2026-02-22T21:45:47.056034+0000","last_became_active":"2026-02-22T21:45:45.053434+0000","last_became_peered":"2026-02-22T21:45:45.053434+0000","last_unstale":"2026-02-22T21:45:47.056034+0000","last_undegraded":"2026-02-22T21:45:47.056034+0000","last_fullsized":"2026-02-22T21:45:47.056034+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:45.436732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076235300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":109,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T21:45:47.055093+0000","last_change":"2026-02-22T21:45:41.044949+0000","last_active":"2026-02-22T21:45:47.055093+0000","last_peered":"2026-02-22T21:45:47.055093+0000","last_clean":"2026-02-22T21:45:47.055093+0000","last_became_active":"2026-02-22T21:45:41.044711+0000","last_became_peered":"2026-02-22T21:45:41.044711+0000","last_unstale":"2026-02-22T21:45:47.055093+0000","last_undegraded":"2026-02-22T21:45:47.055093+0000","last_fullsized":"2026-02-22T21:45:47.055093+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_clean_scrub_stamp":"2026-02-22T21:45:40.040843+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:25:57.694568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27088,"kb_used_data":120,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344752,"statfs":{"total":96636764160,"available":96609026048,"internally_reserved":0,"allocated":122880,"data_stored":33679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27589063},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27680,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344160,"statfs":{"total":96636764160,"available":96608419840,"internally_reserved":0,"allocated":729088,"data_stored":627531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T21:45:50.677 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T21:45:50.677 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-22T21:45:50.677 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-22T21:45:50.677 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-22T21:45:50.849 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T21:45:50.849 INFO:teuthology.orchestra.run.trial104.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T21:45:50.858 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-22T21:45:50.858 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T21:45:50.867 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T21:45:50.867 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:45:50.867 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:51.064 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.4] 2026-02-22T21:45:51.064 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC/eJtp4DfoExAAJJDaDhCHNobsqMVC3HAknw== 2026-02-22T21:45:51.064 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:51.064 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:51.065 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.4 2026-02-22T21:45:51.074 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:51.271 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.5] 2026-02-22T21:45:51.271 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC/eJtp+QPvGBAAm2ZfFewDX+H2VwNL1lps+w== 2026-02-22T21:45:51.271 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:51.271 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:51.271 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.5 2026-02-22T21:45:51.281 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:51.475 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.6] 2026-02-22T21:45:51.476 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC/eJtpLRn2HRAA6ZX1Dx/Y5nIbl8upDlbPfw== 2026-02-22T21:45:51.476 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:51.476 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:51.476 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.6 2026-02-22T21:45:51.485 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:51.678 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.4] 2026-02-22T21:45:51.679 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpkTjNFRAAN2z+1cTQZHLdEKG8ENg9DQ== 2026-02-22T21:45:51.679 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:51.679 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:51.679 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.4 2026-02-22T21:45:51.688 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:51.884 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.5] 2026-02-22T21:45:51.884 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpvn3EGhAAJPSXHG0soFAG2SEcqdEObg== 2026-02-22T21:45:51.884 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:51.884 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:51.884 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.5 2026-02-22T21:45:51.894 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:52.089 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.6] 2026-02-22T21:45:52.089 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtp1kHEHxAABJyjyHqrGVzimPqJB9t/4g== 2026-02-22T21:45:52.090 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:52.090 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:52.090 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.6 2026-02-22T21:45:52.099 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T21:45:52.114 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T21:45:52.114 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial104.front.sepia.ceph.com 2026-02-22T21:45:52.114 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:52.310 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror] 2026-02-22T21:45:52.310 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC+eJtpqVY9NhAAziZqDFCXaT+5A1zpHmFb1g== 2026-02-22T21:45:52.310 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T21:45:52.310 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:52.310 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror 2026-02-22T21:45:52.319 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:52.517 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.0] 2026-02-22T21:45:52.517 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC+eJtpdzJGOxAAOr2AERBYLBCmJN6Ax0mq8w== 2026-02-22T21:45:52.517 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:52.517 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:52.517 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.0 2026-02-22T21:45:52.527 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:52.721 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.1] 2026-02-22T21:45:52.722 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC/eJtpFnG5BBAARvbizJ3RD7ciUL53LGflKA== 2026-02-22T21:45:52.722 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:52.722 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:52.722 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.1 2026-02-22T21:45:52.730 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:52.924 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.2] 2026-02-22T21:45:52.924 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC/eJtp+uG7CRAATEOikjR2MPx5Gm6R3b1cIQ== 2026-02-22T21:45:52.924 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:52.924 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:52.925 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.2 2026-02-22T21:45:52.935 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:53.129 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.3] 2026-02-22T21:45:53.129 INFO:teuthology.orchestra.run.trial104.stdout: key = AQC/eJtpNLfgDhAAdaTWhiicBckydpszzf7VGA== 2026-02-22T21:45:53.129 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:53.129 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:53.129 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.3 2026-02-22T21:45:53.139 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:53.333 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror] 2026-02-22T21:45:53.333 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDneJtpeLADOBAAPtEhZFEAg1PIjt+UTAhvMA== 2026-02-22T21:45:53.333 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T21:45:53.333 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:53.334 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror 2026-02-22T21:45:53.343 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:53.539 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.0] 2026-02-22T21:45:53.540 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpfHmHARAA5M/6tT4KKafpgp0wGxP3Uw== 2026-02-22T21:45:53.540 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:53.540 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:53.540 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.0 2026-02-22T21:45:53.550 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:53.745 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.1] 2026-02-22T21:45:53.745 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpm2GABhAAHPZZzfqwd+VsapU78ruh7A== 2026-02-22T21:45:53.745 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:53.746 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:53.746 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.1 2026-02-22T21:45:53.755 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:53.950 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.2] 2026-02-22T21:45:53.950 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtpkKeeCxAAz0oKYW0N9esgxx0zkJAjGA== 2026-02-22T21:45:53.951 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:53.951 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:53.951 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.2 2026-02-22T21:45:53.961 DEBUG:teuthology.orchestra.run.trial104:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T21:45:54.158 INFO:teuthology.orchestra.run.trial104.stdout:[client.mirror.3] 2026-02-22T21:45:54.158 INFO:teuthology.orchestra.run.trial104.stdout: key = AQDoeJtp8AytEBAAPIB+q093oFq93fwrgEepcA== 2026-02-22T21:45:54.158 INFO:teuthology.orchestra.run.trial104.stdout: caps mon = "profile rbd-mirror" 2026-02-22T21:45:54.158 INFO:teuthology.orchestra.run.trial104.stdout: caps osd = "profile rbd" 2026-02-22T21:45:54.159 INFO:teuthology.orchestra.run.trial104.stderr:updated caps for client.mirror.3 2026-02-22T21:45:54.168 INFO:teuthology.run_tasks:Running task workunit... 2026-02-22T21:45:54.188 INFO:tasks.workunit:Pulling workunits from ref f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:45:54.189 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-22T21:45:54.189 DEBUG:teuthology.orchestra.run.trial104:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T21:45:54.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T21:45:54.206 INFO:teuthology.orchestra.run.trial104.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-22T21:45:54.206 DEBUG:teuthology.orchestra.run.trial104:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T21:45:54.262 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T21:45:54.262 DEBUG:teuthology.orchestra.run.trial104:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-22T21:45:54.316 INFO:tasks.workunit:timeout=3h 2026-02-22T21:45:54.316 INFO:tasks.workunit:cleanup=True 2026-02-22T21:45:54.317 DEBUG:teuthology.orchestra.run.trial104:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T21:45:54.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-22T21:46:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Note: switching to 'f90f4eaebb28b619cf76e80cbbe997656da69579'. 2026-02-22T21:46:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T21:46:15.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:state without impacting any branches by switching back to a branch. 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: git switch -c 2026-02-22T21:46:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T21:46:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Or undo this operation with: 2026-02-22T21:46:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T21:46:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: git switch - 2026-02-22T21:46:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T21:46:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-22T21:46:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T21:46:15.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:HEAD is now at f90f4eaebb2 Merge branch 'wip-74712-squid' of https://github.com/idryomov/ceph into wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T21:46:15.014 DEBUG:teuthology.orchestra.run.trial104:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-22T21:46:15.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-22T21:46:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T21:46:15.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-22T21:46:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-22T21:46:15.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-22T21:46:15.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T21:46:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T21:46:15.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-22T21:46:15.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T21:46:15.252 DEBUG:teuthology.orchestra.run.trial104:> set -ex 2026-02-22T21:46:15.252 DEBUG:teuthology.orchestra.run.trial104:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-22T21:46:15.309 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror.sh on cluster1.client.mirror... 2026-02-22T21:46:15.310 INFO:tasks.workunit:Running workunit rbd/rbd_mirror.sh... 2026-02-22T21:46:15.311 DEBUG:teuthology.orchestra.run.trial104:workunit test rbd/rbd_mirror.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f90f4eaebb28b619cf76e80cbbe997656da69579 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' RBD_MIRROR_INSTANCES=4 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T21:46:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -gt 0 ']' 2026-02-22T21:46:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal '!=' snapshot ']' 2026-02-22T21:46:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal '!=' journal ']' 2026-02-22T21:46:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T21:46:15.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ type xmlstarlet 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ XMLSTARLET=xmlstarlet 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ RBD_MIRROR_INSTANCES=4 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CLUSTER1=cluster1 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CLUSTER2=cluster2 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ POOL=mirror 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ PARENT_POOL=mirror_parent 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ NS1=ns1 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ NS2=ns2 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ TEMPDIR= 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ID=mirror 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ RBD_MIRROR_MODE=journal 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ MIRROR_POOL_MODE=pool 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' journal = snapshot ']' 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS='--id mirror' 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ LAST_MIRROR_INSTANCE=3 2026-02-22T21:46:15.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror.sh 2026-02-22T21:46:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-22T21:46:15.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-22T21:46:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_BIN=. 2026-02-22T21:46:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_SRC=. 2026-02-22T21:46:15.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-22T21:46:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 -gt 0 ']' 2026-02-22T21:46:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ setup 2026-02-22T21:46:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local c 2026-02-22T21:46:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-22T21:46:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ setup_tempdir 2026-02-22T21:46:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:15.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mktemp -d 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ TEMPDIR=/tmp/tmp.S4fo03ccUK 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z 1 ']' 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ setup_pools cluster1 cluster2 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster2 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mon_map_file 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mon_addr 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local admin_key_file 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:15.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-22T21:46:16.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror' already exists 2026-02-22T21:46:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:16.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-22T21:46:17.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror_parent' already exists 2026-02-22T21:46:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:17.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-22T21:46:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:20.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-22T21:46:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2026-02-22T21:46:23.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-22T21:46:23.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-22T21:46:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-22T21:46:23.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-22T21:46:23.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2026-02-22T21:46:23.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-22T21:46:23.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 pool 2026-02-22T21:46:23.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z ']' 2026-02-22T21:46:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z ']' 2026-02-22T21:46:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_add cluster1 mirror cluster2 2026-02-22T21:46:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:23.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local client_cluster=cluster2 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster2 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid_var_name 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local error_code 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local peer_uuid 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:46:23.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror cluster2 2026-02-22T21:46:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_uuid=13c2bf1c-1bf7-47de-a7e8-2f672622352f 2026-02-22T21:46:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:46:23.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 -eq 0 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_add cluster1 mirror_parent cluster2 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local client_cluster=cluster2 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:23.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster2 2026-02-22T21:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid_var_name 2026-02-22T21:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local error_code 2026-02-22T21:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local peer_uuid 2026-02-22T21:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:46:23.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool peer add mirror_parent cluster2 2026-02-22T21:46:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_uuid=a3d45520-9716-48d4-b2de-3d6c472f0f91 2026-02-22T21:46:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:46:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:46:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 -eq 0 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ setup_pools cluster2 cluster1 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster1 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mon_map_file 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mon_addr 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local admin_key_file 2026-02-22T21:46:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid 2026-02-22T21:46:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-22T21:46:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror' already exists 2026-02-22T21:46:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-22T21:46:25.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror_parent' already exists 2026-02-22T21:46:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:25.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-22T21:46:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-22T21:46:31.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:31.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T21:46:31.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2026-02-22T21:46:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-22T21:46:31.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-22T21:46:31.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-22T21:46:31.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-22T21:46:31.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2026-02-22T21:46:31.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-22T21:46:31.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 pool 2026-02-22T21:46:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z ']' 2026-02-22T21:46:31.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z ']' 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_add cluster2 mirror cluster1 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local client_cluster=cluster1 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster1 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid_var_name 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local error_code 2026-02-22T21:46:31.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local peer_uuid 2026-02-22T21:46:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:46:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:46:31.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror cluster1 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_uuid=f705ee6a-c288-4b72-b552-0bc15fdde15b 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 -eq 0 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:31.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_add cluster2 mirror_parent cluster1 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local client_cluster=cluster1 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster1 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid_var_name 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local error_code 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local peer_uuid 2026-02-22T21:46:31.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T21:46:31.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:46:31.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool peer add mirror_parent cluster1 2026-02-22T21:46:31.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_uuid=bc6066fc-3136-4f99-bd73-ea9465e4084f 2026-02-22T21:46:31.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:46:31.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:46:31.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 -eq 0 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: add image and test replay' 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:46:31 TEST: add image and test replay 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:46:31 TEST: add image and test replay 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster1 2026-02-22T21:46:31.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:31.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:0 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:31.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:31.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:31.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:2 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:31.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:31.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:3 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:31.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:31.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:31.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image=test 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test journal 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:46:31.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:31.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test 2026-02-22T21:46:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:46:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:46:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:46:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:46:31.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_image_meta cluster2 mirror test key1 value1 2026-02-22T21:46:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=key1 2026-02-22T21:46:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=value1 2026-02-22T21:46:31.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key1 value1 2026-02-22T21:46:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_image_meta cluster2 mirror test key2 value2 2026-02-22T21:46:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=key2 2026-02-22T21:46:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=value2 2026-02-22T21:46:31.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 image-meta set mirror/test key2 value2 2026-02-22T21:46:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:46:31.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:31.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:32.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:32.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:46:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:46:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:32.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:46:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:32.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:32.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:32.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:32.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:32.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:32.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:46:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:32.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:46:32.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:32.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:32.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:32.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:32.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:46:32.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:46:32.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:32.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:46:32.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:32.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:32.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:46:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:46:32.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:32.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:46:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:46:32.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:32.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:32.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:32.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:32.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:46:32.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:32.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:46:33.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:33.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:33.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:33.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:33.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:33.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:33.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:33.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:46:33.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:33.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:33.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:46:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:46:33.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:33.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:46:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:33.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:33.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:33.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:46:33.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:46:34.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:34.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:34.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:34.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:46:34.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:34.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:46:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:34.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:34.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T21:46:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:36.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:36.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:36.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:36.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:36.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:36.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:36.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:46:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:46:36.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:46:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:36.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:36.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:46:36.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:36.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:36.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:46:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:36.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:36.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:46:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:46:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:36.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:36.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:46:36.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:46:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:46:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:36.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:36.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:46:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:46:36.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:36.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:46:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:46:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:36.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:36.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 4 2026-02-22T21:46:40.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:40.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:40.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:40.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:40.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:46:40.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:46:40.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:46:40.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:46:40.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test 100 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:46:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:46:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:46:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:46:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:46:40.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:46:40.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T21:46:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:46:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:46:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:40.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:46:40.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:40.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:40.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:40.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:40.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:40.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:40.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:46:40.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:40.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:40.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T21:46:40.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:40.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:46:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:46:40.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:46:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:40.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:40.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:40.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T21:46:40.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:40.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=3, entry_tid=3' '!=' '' 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=3, entry_tid=3' = '' 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:46:40.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:40.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=4 2026-02-22T21:46:40.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:40.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:46:40.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:40.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=3 2026-02-22T21:46:40.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:40.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:46:40.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T21:46:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:46:40.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=3 2026-02-22T21:46:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 4 = 3 -a 101 -le 3 2026-02-22T21:46:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:46:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:46:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:46:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:40.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:40.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:40.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:40.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:46:41.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:46:41.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:41.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:41.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T21:46:41.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:46:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:46:41.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:46:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:41.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:41.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:41.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T21:46:41.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=3, entry_tid=3' '!=' 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T21:46:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:41.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T21:46:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:46:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:46:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:46:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:46:41.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:46:41.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:41.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:41.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:41.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:41.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:41.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:41.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:41.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:41.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:41.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:41.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:41.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:41.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:41.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:41.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:41.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=4, entry_tid=101' -a 'object_number=1, tag_tid=4, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:41.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:46:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:41.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:42.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:46:42.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:46:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:46:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:46:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}} 2026-02-22T21:46:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:46:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:46:36' 2026-02-22T21:46:42.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:46:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:46:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:46:42.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}} 2026-02-22T21:46:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:46:42.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:46:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:46:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:46:42.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:46:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:46:42.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:46:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:36' 2026-02-22T21:46:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:46:36' 2026-02-22T21:46:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:42.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:46:36' 2026-02-22T21:46:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:42.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:46:42.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING142644269mirror.0trial10419.2.3-689-gf90f4eaetrueOK42714276mirror.1trial10419.2.3-689-gf90f4eaefalseOK42784283mirror.2trial10419.2.3-689-gf90f4eaefalseOK42864291mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}}42644269mirror.0trial1042026-02-22 21:46:36' 2026-02-22T21:46:42.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:46:42.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:46:42.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:36' 2026-02-22T21:46:42.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:46:42.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:46:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:46:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:46:42.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test down+unknown 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=down+unknown 2026-02-22T21:46:42.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:46:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:42.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test down+unknown '' '' 2026-02-22T21:46:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=down+unknown 2026-02-22T21:46:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:46:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:43.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:46:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:46:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:46:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T21:46:43.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T21:46:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22T21:46:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:46:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:46:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:46:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}} 2026-02-22T21:46:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:46:36' 2026-02-22T21:46:43.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T21:46:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: down+unknown 2026-02-22T21:46:43.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:46:43.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: status not found 2026-02-22T21:46:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo down+unknown 2026-02-22T21:46:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:46:43.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: ' 2026-02-22T21:46:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:46:43.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:46:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update= 2026-02-22T21:46:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test down+unknown '' 2026-02-22T21:46:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:43.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=down+unknown 2026-02-22T21:46:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local prev_last_update= 2026-02-22T21:46:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:43.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:46:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='UNKNOWNUNKNOWNOK1test767d454f-67ec-4a4a-999f-9cdd378268dcdown+unknownstatus not foundcluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}}2026-02-22 21:46:36' 2026-02-22T21:46:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:46:43.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:46:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update= 2026-02-22T21:46:43.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:46:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=down+unknown 2026-02-22T21:46:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo down+unknown 2026-02-22T21:46:43.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep down+unknown 2026-02-22T21:46:43.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:down+unknown 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T21:46:43.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:46:43.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T21:46:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T21:46:43.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:43.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:43.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:43.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:46:44.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:44.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:44.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:44.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:44.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_image_meta cluster1 mirror test key1 value1 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=key1 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local value=value1 2026-02-22T21:46:44.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key1 2026-02-22T21:46:44.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test value1 = value1 2026-02-22T21:46:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_image_meta cluster1 mirror test key2 value2 2026-02-22T21:46:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=key2 2026-02-22T21:46:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local value=value2 2026-02-22T21:46:44.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 image-meta get mirror/test key2 2026-02-22T21:46:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test value2 = value2 2026-02-22T21:46:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: stop mirror, add image, start mirror and test replay' 2026-02-22T21:46:44.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:46:44.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:46:44.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:46:44 TEST: stop mirror, add image, start mirror and test replay 2026-02-22T21:46:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:46:44 TEST: stop mirror, add image, start mirror and test replay 2026-02-22T21:46:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster1 2026-02-22T21:46:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T21:46:44.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:44.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:0 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:44.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:44.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T21:46:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T21:46:44.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T21:46:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=49388 2026-02-22T21:46:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 49388 ']' 2026-02-22T21:46:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 49388 2026-02-22T21:46:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:46:44.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:44.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:44.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49388 '$2 == pid {print; exit 1}' 2026-02-22T21:46:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 49388 1.0 0.0 0 0 ? Rs 21:46 0:00 [rbd-mirror] 2026-02-22T21:46:44.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:46:44.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:45.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49388 '$2 == pid {print; exit 1}' 2026-02-22T21:46:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:46:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:45.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49388 '$2 == pid {print; exit 1}' 2026-02-22T21:46:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:45.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:45.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:0 2026-02-22T21:46:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T21:46:45.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T21:46:45.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:1 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:45.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:46:45.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T21:46:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T21:46:45.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T21:46:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=49431 2026-02-22T21:46:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 49431 ']' 2026-02-22T21:46:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 49431 2026-02-22T21:46:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:46:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49431 '$2 == pid {print; exit 1}' 2026-02-22T21:46:45.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:46:45.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:45.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49431 '$2 == pid {print; exit 1}' 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:45.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:45.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:45.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T21:46:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:46:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:45.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:45.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:45.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:1 2026-02-22T21:46:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:45.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:45.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T21:46:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T21:46:45.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:2 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:46:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T21:46:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T21:46:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T21:46:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=49490 2026-02-22T21:46:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 49490 ']' 2026-02-22T21:46:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 49490 2026-02-22T21:46:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:46:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49490 '$2 == pid {print; exit 1}' 2026-02-22T21:46:45.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:46:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:45.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49490 '$2 == pid {print; exit 1}' 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:45.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:45.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:45.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:45.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:45.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:45.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:2 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:45.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T21:46:45.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T21:46:45.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T21:46:45.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:45.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T21:46:45.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:3 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:45.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T21:46:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T21:46:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:46:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:46:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T21:46:45.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T21:46:45.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T21:46:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=49552 2026-02-22T21:46:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 49552 ']' 2026-02-22T21:46:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 49552 2026-02-22T21:46:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:46:45.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:45.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:45.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49552 '$2 == pid {print; exit 1}' 2026-02-22T21:46:45.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 49552 0.6 0.0 0 0 ? Rs 21:46 0:00 [rbd-mirror] 2026-02-22T21:46:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:46:45.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49552 '$2 == pid {print; exit 1}' 2026-02-22T21:46:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:46:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T21:46:46.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=49552 '$2 == pid {print; exit 1}' 2026-02-22T21:46:46.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:46.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:46.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:46.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T21:46:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:46:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:46.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:46.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:3 2026-02-22T21:46:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:46:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T21:46:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T21:46:46.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image1=test1 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test1 journal 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:46:46.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test1 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:46.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test1 2026-02-22T21:46:46.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:46:46.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:46.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:46:46.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test1 100 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:46:46.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:46:46.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:46:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:46:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 9091.09 bytes/sec: 36 MiB/s 2026-02-22T21:46:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster1 2026-02-22T21:46:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:46.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:0 2026-02-22T21:46:46.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:46.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:46.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:46.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:46.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:1 2026-02-22T21:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:46:46.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:46.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:46.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:2 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:46.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:46:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:46.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:46.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:3 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:46:46.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:46:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:46.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:46:46.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:46.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T21:46:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:46:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:46:47.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:46:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:46:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:46:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:46:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:46:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:46:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:46:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:47.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:47.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:47.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:47.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:47.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:47.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:46:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:48.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:46:48.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:46:48.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T21:46:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:48.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:48.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:48.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:46:48.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T21:46:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:48.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:48.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:48.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:46:48.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:48.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:48.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:48.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:48.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-02-22T21:46:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:48.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:48.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:46:49.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:49.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:49.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:49.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:49.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:49.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:49.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:49.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:46:49.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:49.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test1 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:49.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:49.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:46:49.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:49.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test1 2026-02-22T21:46:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test1 2026-02-22T21:46:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:46:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:49.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:49.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:46:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:46:49.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test1 2026-02-22T21:46:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:49.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test1 2026-02-22T21:46:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:46:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:49.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:49.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:46:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:46:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:49.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test1 2026-02-22T21:46:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test1'] 2026-02-22T21:46:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:46:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:46:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:49.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T21:46:51.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:51.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:51.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:51.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:46:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:46:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:46:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:46:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:46:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test1 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test1 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:46:51.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:51.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test1 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test1 ']' 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test1' 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test1 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:51.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:51.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test1 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:51.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:51.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:51.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test1 2026-02-22T21:46:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test1 2026-02-22T21:46:51.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test1 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test1 '' 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test1 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T21:46:51.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.status 2026-02-22T21:46:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T21:46:51.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.status 2026-02-22T21:46:51.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:46:51.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:46:51.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.status 2026-02-22T21:46:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:46:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test1 2026-02-22T21:46:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:51.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test1 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test1 '..*' 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test1 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test1.status 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test1.status 2026-02-22T21:46:51.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.status 2026-02-22T21:46:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test1 2026-02-22T21:46:51.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.status 2026-02-22T21:46:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:46:51.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:46:51.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.status 2026-02-22T21:46:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:46:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:46:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test1 2026-02-22T21:46:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:51.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:51.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying primary_position '' 2026-02-22T21:46:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:46:52.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:52.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:52.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:52.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:46:52.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:46:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:46:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:46:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:46:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:46:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:46:50' 2026-02-22T21:46:52.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:46:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:46:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:46:52.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:46:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:46:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:46:52.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:46:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:46:52.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:46:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:46:52.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:46:52.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:50' 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 21:46:50' 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:46:50' 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:52.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:46:52.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43184322mirror.0trial1042026-02-22 21:46:51test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:46:50' 2026-02-22T21:46:52.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:46:52.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:46:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:50' 2026-02-22T21:46:52.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:46:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:46:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:46:52.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:46:52.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test1 down+unknown 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:52.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=down+unknown 2026-02-22T21:46:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:46:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:52.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test1 down+unknown '' '' 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=down+unknown 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:53.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test1 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:46:53.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:46:50' 2026-02-22T21:46:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*down+unknown' 2026-02-22T21:46:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: down+unknown 2026-02-22T21:46:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:46:53.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: status not found 2026-02-22T21:46:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo down+unknown 2026-02-22T21:46:53.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:46:53.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: ' 2026-02-22T21:46:53.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:46:53.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update= 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test1 down+unknown '' 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=down+unknown 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local prev_last_update= 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:53.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:46:53.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='UNKNOWNUNKNOWNOK2test767d454f-67ec-4a4a-999f-9cdd378268dcdown+unknownstatus not foundcluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}2026-02-22 21:46:51test11579a376-a16c-4a27-ad76-944c0ac73ee7down+unknownstatus not foundcluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}2026-02-22 21:46:50' 2026-02-22T21:46:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:46:53.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:46:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update= 2026-02-22T21:46:53.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:46:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=down+unknown 2026-02-22T21:46:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo down+unknown 2026-02-22T21:46:53.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep down+unknown 2026-02-22T21:46:53.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:down+unknown 2026-02-22T21:46:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:53.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test1 2026-02-22T21:46:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:46:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test1.export 2026-02-22T21:46:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test1.export 2026-02-22T21:46:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:53.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.export 2026-02-22T21:46:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test1.export 2026-02-22T21:46:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test1.export 2026-02-22T21:46:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:53.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test1.export 2026-02-22T21:46:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test1.export 2026-02-22T21:46:53.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test1 /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.export 2026-02-22T21:46:54.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:54.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test1 /tmp/tmp.S4fo03ccUK/cluster1-mirror-test1.export 2026-02-22T21:46:54.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test1.export 2026-02-22T21:46:54.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test1.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test1.export 2026-02-22T21:46:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:54.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: test the first image is replaying after restart' 2026-02-22T21:46:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:46:54.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:46:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:46:54 TEST: test the first image is replaying after restart 2026-02-22T21:46:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:46:54 TEST: test the first image is replaying after restart 2026-02-22T21:46:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test 100 2026-02-22T21:46:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:54.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:46:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:46:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:46:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:46:54.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:46:54.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:46:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:46:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 5555.67 bytes/sec: 22 MiB/s 2026-02-22T21:46:54.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:46:54.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:54.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:55.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:46:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:46:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:46:55.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:55.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:46:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:55.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:55.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:55.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:55.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:55.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:55.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:46:55.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:46:55.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:46:55.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:46:55.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:46:55.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:46:55.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:55.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:46:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:46:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:55.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:55.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:55.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:55.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:55.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:46:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:55.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:55.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T21:46:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:46:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:55.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' '' 2026-02-22T21:46:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:46:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' = '' 2026-02-22T21:46:55.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T21:46:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:46:55.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:55.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=5 2026-02-22T21:46:55.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:46:55.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:55.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=4 2026-02-22T21:46:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T21:46:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:46:55.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T21:46:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:46:55.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:46:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=101 2026-02-22T21:46:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 5 = 4 -a 101 -le 101 2026-02-22T21:46:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:46:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:55.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:55.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:55.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:55.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:55.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:55.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:55.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:46:55.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:46:55.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:55.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:55.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:55.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T21:46:55.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:46:55.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:55.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:55.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:55.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:55.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=4, entry_tid=101], [object_number=0, tag_tid=4, entry_tid=100], [object_number=3, tag_tid=4, entry_tid=99], [object_number=2, tag_tid=4, entry_tid=98]]], state=connected] 2026-02-22T21:46:55.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=4, entry_tid=101' '!=' 'object_number=1, tag_tid=4, entry_tid=101' 2026-02-22T21:46:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:46:55.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T21:46:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:46:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:56.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:56.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:56.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:46:56.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:56.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:56.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:56.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:56.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:56.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:56.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:56.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:56.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:56.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:46:56.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:56.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:56.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:56.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:56.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:56.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:56.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:56.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T21:46:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:46:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:46:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:46:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:46:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:46:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:46:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T21:46:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:46:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T21:46:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T21:46:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T21:46:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:46:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T21:46:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:46:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:57.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:46:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:46:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:46:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:46:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T21:46:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:46:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:46:51' 2026-02-22T21:46:57.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:46:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:46:57.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:46:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}} 2026-02-22T21:46:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:46:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:46:57.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:46:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:46:57.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:46:57.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:46:57.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:46:57.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:51' 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:46:51' 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:46:51' 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:46:57.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:46:57.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":4}}43184322mirror.0trial1042026-02-22 21:46:51test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:46:50' 2026-02-22T21:46:57.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:46:57.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:46:57.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:51' 2026-02-22T21:46:57.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:46:57.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:46:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:46:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:46:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:46:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T21:46:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:57.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:46:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T21:46:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T21:46:57.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:46:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T21:46:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T21:46:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:46:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:57.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:46:57.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:57.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:57.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:46:57.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:57.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:46:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:57.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T21:46:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: stop/start/restart mirror via admin socket' 2026-02-22T21:46:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:46:57.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:46:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:46:57 TEST: stop/start/restart mirror via admin socket 2026-02-22T21:46:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:46:57 TEST: stop/start/restart mirror via admin socket 2026-02-22T21:46:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-22T21:46:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:57.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:46:57.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:57.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:57.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:57.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:46:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:46:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:46:57.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-22T21:46:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-22T21:46:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:46:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:57.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:46:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:46:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:57.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:46:57.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-22T21:46:58.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-22T21:46:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:46:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:46:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:46:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:46:58.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:46:58.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:46:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:46:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:46:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:46:58.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-22T21:46:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:46:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:58.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:46:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:46:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:46:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:58.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:46:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:46:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:46:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:46:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:46:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:59.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:46:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:46:59.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:46:59.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:46:59.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:46:59.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:59.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:59.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:46:59.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:46:59.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:46:59.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:46:59.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:46:59.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:46:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:46:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:46:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T21:46:59.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:46:59.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:00.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:47:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:47:00.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:00.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:01.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:46:57' 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:01.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:01.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:01.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:01.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:01.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:01.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:01.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:01.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:01.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:01.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:01.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:57' 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:46:57' 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:46:57' 2026-02-22T21:47:01.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:01.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:46:57test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:46:57' 2026-02-22T21:47:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:57' 2026-02-22T21:47:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:01.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:01.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T21:47:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:02.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:46:57' 2026-02-22T21:47:02.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:02.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:02.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:02.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:02.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:02.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:02.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:02.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:57' 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 21:46:57' 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:46:57' 2026-02-22T21:47:02.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:02.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:46:57test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:46:57' 2026-02-22T21:47:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:46:57' 2026-02-22T21:47:02.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror start 2026-02-22T21:47:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror start 2026-02-22T21:47:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:02.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:02.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start 2026-02-22T21:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror start 2026-02-22T21:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:02.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:02.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:02.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:02.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:02.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start 2026-02-22T21:47:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:02.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror start 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:02.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:02.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror start 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:02.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:02.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:02.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:02.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:02.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:03.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:03.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:03.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:03.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:03.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:03.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:03.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:03.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:03.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:03.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:03.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:03.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T21:47:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:03.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:03.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:04.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:04.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:04.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:04.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:04.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:04.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:04.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:04.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:05.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:02' 2026-02-22T21:47:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:05.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:05.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:05.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:05.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:05.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:02' 2026-02-22T21:47:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:47:02' 2026-02-22T21:47:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:05.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:02' 2026-02-22T21:47:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:05.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:05.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:02test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:02' 2026-02-22T21:47:05.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:05.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:02' 2026-02-22T21:47:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T21:47:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:05.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:06.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:02' 2026-02-22T21:47:06.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:06.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:06.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:06.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:06.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:06.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:06.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:06.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:02' 2026-02-22T21:47:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 21:47:02' 2026-02-22T21:47:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:06.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:02' 2026-02-22T21:47:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:06.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:06.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:02test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:02' 2026-02-22T21:47:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:06.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:02' 2026-02-22T21:47:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:06.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:06.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:06.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:06.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:06.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:06.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-22T21:47:06.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:06.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:06.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:06.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:06.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-22T21:47:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-22T21:47:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:07.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:07.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:07.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:07.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:07.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:07.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:07.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:07.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:07.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:07.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:07.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:07.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-22T21:47:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:07.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:08.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:08.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:08.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:08.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:08.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:08.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:08.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:08.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:08.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:08.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:08.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:08.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:09.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:09.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:09.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:09.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:09.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:09.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:09.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:09.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:09.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:09.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:09.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:47:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:10.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:10.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:06' 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:10.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:10.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:10.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:10.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:10.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:10.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:10.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:06' 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:47:06' 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:06' 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:10.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:06test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:06' 2026-02-22T21:47:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:10.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:06' 2026-02-22T21:47:10.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:10.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:10.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:10.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:10.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:11.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:11.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:06' 2026-02-22T21:47:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:11.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:11.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:11.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:11.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:11.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:11.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:06' 2026-02-22T21:47:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 21:47:06' 2026-02-22T21:47:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:11.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:06' 2026-02-22T21:47:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:11.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:06test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:06' 2026-02-22T21:47:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:11.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:11.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:06' 2026-02-22T21:47:11.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:11.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:11.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror stop 2026-02-22T21:47:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:11.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror stop 2026-02-22T21:47:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:11.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:11.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:11.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:11.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:11.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:11.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop 2026-02-22T21:47:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror stop 2026-02-22T21:47:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:11.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:11.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:11.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:11.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:11.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:11.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:11.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop 2026-02-22T21:47:11.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:11.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror stop 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:11.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:11.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:11.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:11.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:11.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:11.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:11.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:11.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror stop 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:11.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:11.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:11.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:12.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:12.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:12.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:47:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:47:12.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:12.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:47:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:47:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:13.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:13.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:13.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:13.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:13.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:13.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:13.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:13.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:13.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:47:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:47:13.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:13.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:13.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:14.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:11' 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:14.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:14.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:14.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:14.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:14.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:11' 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:47:11' 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:11' 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:14.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:11test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:11' 2026-02-22T21:47:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:14.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:11' 2026-02-22T21:47:14.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:14.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:14.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:14.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:15.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:15.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:15.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:11' 2026-02-22T21:47:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:15.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:11' 2026-02-22T21:47:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 21:47:11' 2026-02-22T21:47:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:11' 2026-02-22T21:47:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:11test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:11' 2026-02-22T21:47:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:15.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:11' 2026-02-22T21:47:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:15.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:15.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror restart 2026-02-22T21:47:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:15.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:15.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror restart 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:15.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:15.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:15.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:15.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:15.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart 2026-02-22T21:47:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror restart 2026-02-22T21:47:15.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:15.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:15.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:15.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:16.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror restart 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:16.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart 2026-02-22T21:47:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror restart 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:16.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:16.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:16.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:16.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart 2026-02-22T21:47:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:17.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:17.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:17.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:17.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:17.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:17.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:17.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:17.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:17.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:17.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:17.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:18.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:18.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:18.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:18.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:18.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:18.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:18.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:18.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:18.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:47:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:18.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:18.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:19.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:19.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:16' 2026-02-22T21:47:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:19.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:16' 2026-02-22T21:47:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:47:16' 2026-02-22T21:47:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:19.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:16' 2026-02-22T21:47:19.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:19.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:19.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:16test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:16' 2026-02-22T21:47:19.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:16' 2026-02-22T21:47:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:19.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:19.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:19.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:19.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:20.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:20.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:16' 2026-02-22T21:47:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:20.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:20.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:20.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:20.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:16' 2026-02-22T21:47:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 21:47:16' 2026-02-22T21:47:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:20.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:20.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:16' 2026-02-22T21:47:20.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:20.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:20.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:16test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:16' 2026-02-22T21:47:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:20.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:16' 2026-02-22T21:47:20.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:20.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:20.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:20.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2 2026-02-22T21:47:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2 2026-02-22T21:47:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:20.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:20.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:20.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:20.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:20.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:20.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:20.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:20.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:20.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:20.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:20.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:20.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:20.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2 2026-02-22T21:47:20.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:20.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:20.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:20.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:20.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:20.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:20.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:20.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:20.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:20.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:47:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:47:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:21.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:21.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:21.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:21.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:21.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:21.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:21.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:21.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:47:21.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:47:21.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:47:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:21.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:21.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:47:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:22.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:47:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:22.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:22.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:22.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:22.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:22.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:22.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:22.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:47:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:47:22.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:47:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:22.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:22.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:23.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:23.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:23.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:20' 2026-02-22T21:47:23.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:23.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:23.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:23.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:23.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:20' 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:47:20' 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:20' 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:20test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:20' 2026-02-22T21:47:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:20' 2026-02-22T21:47:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:23.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:23.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:24.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:24.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:20' 2026-02-22T21:47:24.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:24.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:24.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:20' 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 21:47:20' 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:20' 2026-02-22T21:47:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:20test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:20' 2026-02-22T21:47:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:24.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:20' 2026-02-22T21:47:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:24.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:24.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:24.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:24.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T21:47:24.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:24.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:24.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:24.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:24.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:24.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:24.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:24.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:25.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:26.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:26.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:26.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:26.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:26.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:26.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:26.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:26.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:26.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:47:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:27.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:27.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:27.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:25' 2026-02-22T21:47:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:27.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:27.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:27.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:27.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:27.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:27.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:27.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:25' 2026-02-22T21:47:27.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:47:25' 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:25' 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:27.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:27.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:25test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:20' 2026-02-22T21:47:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:27.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:25' 2026-02-22T21:47:27.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:27.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:27.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:27.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror start mirror cluster2 2026-02-22T21:47:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:27.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror cluster2 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:27.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:27.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror cluster2 2026-02-22T21:47:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror cluster2 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:27.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:27.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:27.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror cluster2 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror start mirror cluster2 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:27.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:27.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:27.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror start mirror cluster2 2026-02-22T21:47:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror start mirror cluster2 2026-02-22T21:47:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:27.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:27.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:27.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:27.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:27.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:27.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror start mirror cluster2 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:27.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:27.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:28.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:28.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:28.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:28.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:28.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:28.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:28.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:28.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:28.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:28.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:28.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:28.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:28.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T21:47:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:29.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:29.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:27' 2026-02-22T21:47:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:29.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:29.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:29.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:27' 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 21:47:27' 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:27' 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:29.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:25test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:27' 2026-02-22T21:47:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:29.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:27' 2026-02-22T21:47:29.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:29.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test 2026-02-22T21:47:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:47:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:29.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:47:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:47:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:29.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test 2026-02-22T21:47:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:29.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:29.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:29.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test 2026-02-22T21:47:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:29.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:29.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:30.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:30.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:30.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:30.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:30.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:30.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:30.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:30.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:30.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:30.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:30.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:30.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:30.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:30.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:30.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:30.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:30.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:30.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:31.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:31.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:31.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:29' 2026-02-22T21:47:31.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:31.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:31.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:31.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:29' 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:47:29' 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:29' 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:29test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:27' 2026-02-22T21:47:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:31.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:31.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:29' 2026-02-22T21:47:31.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:31.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:31.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:31.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2 2026-02-22T21:47:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2 2026-02-22T21:47:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:31.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:31.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2 2026-02-22T21:47:32.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:32.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:32.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:32.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:32.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:32.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:32.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:32.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2 2026-02-22T21:47:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:32.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:32.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:32.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:32.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:32.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:33.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:33.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:33.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:33.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:33.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:33.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:33.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T21:47:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:33.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:34.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:34.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:34.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:34.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:34.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:34.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:34.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:34.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:34.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:34.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:34.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:34.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:34.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:34.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:34.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:34.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:34.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:34.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:34.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:34.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:34.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:34.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:34.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:34.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:34.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:34.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:34.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:34.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror stop mirror cluster2 2026-02-22T21:47:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:34.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror cluster2 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:34.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:34.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:34.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:34.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror cluster2 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:34.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:34.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:34.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror stop mirror cluster2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:34.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:34.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:34.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:34.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:34.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:34.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:34.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror stop mirror cluster2 2026-02-22T21:47:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:34.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:34.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:34.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:34.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:34.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:34.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:34.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror stop mirror cluster2 2026-02-22T21:47:34.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:47:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:47:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:34.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:34.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:34.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:47:34.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:34.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:34.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:47:35.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:47:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:35.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:35.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:35.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:35.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:35.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:35.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:35.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:35.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:35.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:35.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:35.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:35.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:35.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:35.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:35.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:35.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:35.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:35.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:35.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:35.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:35.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:35.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:35.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:35.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:35.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:35.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:35.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:35.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:35.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:47:35.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:47:35.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:35.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:47:35.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test1 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:35.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:35.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:36.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 stopped 2026-02-22T21:47:36.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:47:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:36.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:36.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:36.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:36.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:36.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:36.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:36.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:36.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:36.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:36.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:36.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:36.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:36.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:36.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:36.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:36.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:47:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:47:36.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:36.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:36.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:36.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:47:37.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:37.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:34' 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:37.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:37.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:37.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:37.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:37.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:34' 2026-02-22T21:47:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:47:34' 2026-02-22T21:47:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:37.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:34' 2026-02-22T21:47:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:37.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:34test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:34' 2026-02-22T21:47:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:37.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:34' 2026-02-22T21:47:37.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:37.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:37.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+stopped 2026-02-22T21:47:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:37.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:37.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+stopped '' '' 2026-02-22T21:47:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:38.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:38.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:34' 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:47:38.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:47:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:47:38.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:38.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:38.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:38.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:34' 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+stopped '2026-02-22 21:47:34' 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:34' 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:38.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:34test11579a376-a16c-4a27-ad76-944c0ac73ee7up+stoppedstopped43184322mirror.0trial1042026-02-22 21:47:34' 2026-02-22T21:47:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:38.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:34' 2026-02-22T21:47:38.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:38.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:47:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:47:38.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:47:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:47:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:38.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror restart mirror cluster2 2026-02-22T21:47:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:38.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror cluster2 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:38.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:38.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:38.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:38.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:38.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:38.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror restart mirror cluster2 2026-02-22T21:47:38.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:38.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:38.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:38.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:38.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:38.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:38.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:38.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror restart mirror cluster2 2026-02-22T21:47:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:38.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:38.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:38.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:38.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:39.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror restart mirror cluster2 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:39.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:39.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:39.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:39.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror restart mirror cluster2 2026-02-22T21:47:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:39.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:39.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:40.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:40.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:40.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:40.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:40.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:40.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:40.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:40.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:40.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:40.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:40.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:40.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:40.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test1 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:40.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:40.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test1 started 2026-02-22T21:47:41.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:41.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test1 2026-02-22T21:47:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:41.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test1 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:41.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:41.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:41.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:41.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:41.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:41.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test1 2026-02-22T21:47:41.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:41.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:41.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:41.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:47:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:42.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:42.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:47:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:47:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:38' 2026-02-22T21:47:42.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:42.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}} 2026-02-22T21:47:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:42.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:42.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:42.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:38' 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:47:38' 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:38' 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:42.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:38test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:38' 2026-02-22T21:47:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:42.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:47:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:38' 2026-02-22T21:47:42.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:47:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:42.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:42.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test1 up+replaying 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:42.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:42.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test1 up+replaying '' '' 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:43.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test1 2026-02-22T21:47:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test1: 2026-02-22T21:47:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 1579a376-a16c-4a27-ad76-944c0ac73ee7 2026-02-22T21:47:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:43.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:38' 2026-02-22T21:47:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:43.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:47:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:43.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:43.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:43.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:38' 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test1 up+replaying '2026-02-22 21:47:38' 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:38' 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:43.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:43.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:47:38test11579a376-a16c-4a27-ad76-944c0ac73ee7up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:47:38' 2026-02-22T21:47:43.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:43.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/last_update' 2026-02-22T21:47:43.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:38' 2026-02-22T21:47:43.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test1'\'']/state' 2026-02-22T21:47:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:43.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:43.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:43.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool= 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image= 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush 2026-02-22T21:47:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:43.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:43.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:43.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush 2026-02-22T21:47:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ all_admin_daemons cluster1 rbd mirror status 2026-02-22T21:47:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:43.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:43.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror status 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:43.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:43.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:43.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:43.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status 2026-02-22T21:47:43.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T21:47:43.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool_replayers": [ 2026-02-22T21:47:43.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: 13c2bf1c-1bf7-47de-a7e8-2f672622352f cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror", 2026-02-22T21:47:43.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4322", 2026-02-22T21:47:43.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4322", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": true, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instances": [ 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4322", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4328", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4335", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4344" 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ], 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.50528.cluster1.93997470382440.asok", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50528.cluster2.93997483358568.asok", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [ 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "mirror/test1", 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "mirror/test", 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ], 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter": { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter_status": { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "delete_images_queue": [], 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "failed_deletes_queue": [] 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter": { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter_status": { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "delete_images_queue": [], 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "failed_deletes_queue": [] 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns2", 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter": { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter_status": { 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "delete_images_queue": [], 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "failed_deletes_queue": [] 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: a3d45520-9716-48d4-b2de-3d6c472f0f91 cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror_parent", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4337", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4337", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": true, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instances": [ 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4337", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4343", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4347", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "4349" 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ], 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.0.50528.cluster1.93997470808424.asok", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50528.cluster2.93997472741736.asok", 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter": { 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter_status": { 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "delete_images_queue": [], 2026-02-22T21:47:43.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "failed_deletes_queue": [] 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter": { 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_deleter_status": { 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "delete_images_queue": [], 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "failed_deletes_queue": [] 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T21:47:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror status 2026-02-22T21:47:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T21:47:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:43.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:43.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:43.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status 2026-02-22T21:47:43.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool_replayers": [ 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: 13c2bf1c-1bf7-47de-a7e8-2f672622352f cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror", 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4328", 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4322", 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": false, 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.50571.cluster1.94090423384424.asok", 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50571.cluster2.94090436327784.asok", 2026-02-22T21:47:43.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns2", 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: a3d45520-9716-48d4-b2de-3d6c472f0f91 cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror_parent", 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4343", 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4337", 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": false, 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.1.50571.cluster1.94090426923368.asok", 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50571.cluster2.94090428766568.asok", 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror status 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T21:47:43.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:47:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:47:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T21:47:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:43.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status 2026-02-22T21:47:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T21:47:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool_replayers": [ 2026-02-22T21:47:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: 13c2bf1c-1bf7-47de-a7e8-2f672622352f cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror", 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4335", 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4322", 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": false, 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.50633.cluster1.94648061679976.asok", 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50633.cluster2.94648074606952.asok", 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns2", 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: a3d45520-9716-48d4-b2de-3d6c472f0f91 cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror_parent", 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4347", 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4337", 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": false, 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.2.50633.cluster1.94648065259880.asok", 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50633.cluster2.94648067078504.asok", 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror status 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:43.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:43.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status 2026-02-22T21:47:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T21:47:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool_replayers": [ 2026-02-22T21:47:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: 13c2bf1c-1bf7-47de-a7e8-2f672622352f cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror", 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4344", 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4322", 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": false, 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.50700.cluster1.94098147154280.asok", 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50700.cluster2.94098147432808.asok", 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns2", 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "peer": "uuid: a3d45520-9716-48d4-b2de-3d6c472f0f91 cluster: cluster2 client: client.mirror", 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "pool": "mirror_parent", 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "instance_id": "4349", 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "stopped (manual)", 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader_instance_id": "4337", 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "leader": false, 2026-02-22T21:47:43.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "local_cluster_admin_socket": "/var/run/ceph/client.mirror.3.50700.cluster1.94098150652264.asok", 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "remote_cluster_admin_socket": "/var/run/ceph/client.mirror.50700.cluster2.94098152536424.asok", 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "sync_throttler": { 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 5, 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "deletion_throttler": { 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "max_parallel_requests": 1, 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "running_requests": 0, 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "waiting_requests": 0 2026-02-22T21:47:43.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: }, 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [], 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "namespaces": [ 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: { 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "ns1", 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "image_replayers": [] 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: } 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: ] 2026-02-22T21:47:43.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T21:47:43.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test1 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test1 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test1 2026-02-22T21:47:43.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test1 2026-02-22T21:47:43.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test1 2026-02-22T21:47:43.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:47:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:43.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: test image rename' 2026-02-22T21:47:43.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:47:43.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:47:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:47:43 TEST: test image rename 2026-02-22T21:47:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:47:43 TEST: test image rename 2026-02-22T21:47:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ new_name=test_RENAMED 2026-02-22T21:47:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_image cluster2 mirror test test_RENAMED 2026-02-22T21:47:43.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_name=test_RENAMED 2026-02-22T21:47:43.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rename mirror/test mirror/test_RENAMED 2026-02-22T21:47:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T21:47:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-22T21:47:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T21:47:43.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:43.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T21:47:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:44.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:44.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:44.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:44.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:44.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T21:47:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:44.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:44.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:47:44.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:44.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T21:47:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:44.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:47:44.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:47:44.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T21:47:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:44.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:47:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:44.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:47:44.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:47:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:44.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_RENAMED'] 2026-02-22T21:47:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:47:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:46.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:46.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:46.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:46.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:46.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:46.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-22T21:47:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:47.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:47.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-22T21:47:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_RENAMED: 2026-02-22T21:47:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6}} 2026-02-22T21:47:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:46' 2026-02-22T21:47:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:47.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:47.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6}} 2026-02-22T21:47:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:47.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:47.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:47.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:46' 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-22 21:47:46' 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:46' 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:47.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:47.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_RENAMED767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":5},"primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6}}43184322mirror.0trial1042026-02-22 21:47:46' 2026-02-22T21:47:47.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:47.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-22T21:47:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:46' 2026-02-22T21:47:47.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-22T21:47:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:47.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:47:47.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:47:47.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:47.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:47.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:47.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:47.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:47.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:47.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:47.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T21:47:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "mirror/test_RENAMED", 2026-02-22T21:47:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:47.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror restart mirror/test_RENAMED 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:47:47.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:47.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:47:47.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror restart mirror/test_RENAMED 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:47.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:47.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:47.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:47.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:47.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror restart mirror/test_RENAMED 2026-02-22T21:47:47.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:47.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_RENAMED 2026-02-22T21:47:47.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:47.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:47.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:47.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T21:47:47.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:47.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:47.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:47.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:47.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:47.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:47.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_RENAMED started 2026-02-22T21:47:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:48.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:48.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:48.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:48.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:48.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:48.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:48.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:48.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:48.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:48.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:48.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_RENAMED up+replaying '' '' 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:49.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_RENAMED 2026-02-22T21:47:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_RENAMED: 2026-02-22T21:47:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 767d454f-67ec-4a4a-999f-9cdd378268dc 2026-02-22T21:47:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:47:49.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6}} 2026-02-22T21:47:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:47:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:47:47' 2026-02-22T21:47:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:47:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:47:49.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:47:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6}} 2026-02-22T21:47:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:49.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:47:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:49.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:47:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:47:49.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:47:49.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:47:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:47' 2026-02-22T21:47:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_RENAMED up+replaying '2026-02-22 21:47:47' 2026-02-22T21:47:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:47:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:47:47' 2026-02-22T21:47:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:47:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:47:49.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:47:49.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_RENAMED767d454f-67ec-4a4a-999f-9cdd378268dcup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6},"primary_position":{"entry_tid":1,"object_number":1,"tag_tid":6}}43184322mirror.0trial1042026-02-22 21:47:47' 2026-02-22T21:47:49.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:47:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/last_update' 2026-02-22T21:47:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:47:47' 2026-02-22T21:47:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_RENAMED'\'']/state' 2026-02-22T21:47:49.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:47:49.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:47:49.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_image cluster2 mirror test_RENAMED test 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_RENAMED 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_name=test 2026-02-22T21:47:49.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rename mirror/test_RENAMED mirror/test 2026-02-22T21:47:49.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T21:47:49.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:49.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:49.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:49.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:49.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:49.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:49.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:49.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:49.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:49.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:49.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:49.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:50.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:50.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:50.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:50.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:50.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:50.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:50.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:50.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:50.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:50.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:50.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:50.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:50.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:50.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:50.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:no valid command found; 1 closest matches: 2026-02-22T21:47:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status mirror/test_RENAMED 2026-02-22T21:47:50.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:50.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:50.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:47:50.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:50.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:50.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:47:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:50.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:50.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:50.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:47:50.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:50.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:50.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:47:50.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:50.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:50.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:47:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:47:50.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:50.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:50.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:50.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:47:50.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:47:50.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:50.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:47:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:50.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:50.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:50.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:47:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:51.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:51.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:51.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:51.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:51.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:51.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:51.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:51.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: test trash move restore' 2026-02-22T21:47:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:47:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:47:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:47:51 TEST: test trash move restore 2026-02-22T21:47:51.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:47:51 TEST: test trash move restore 2026-02-22T21:47:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster2 mirror test 2026-02-22T21:47:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:47:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:47:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster2 info mirror/test 2026-02-22T21:47:51.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id=10b9c75d983f 2026-02-22T21:47:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ trash_move cluster2 mirror test 2026-02-22T21:47:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:51.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 trash move mirror/test 2026-02-22T21:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-22T21:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T21:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T21:47:51.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T21:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T21:47:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T21:47:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T21:47:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T21:47:51.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted '' 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T21:47:51.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T21:47:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T21:47:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:47:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T21:47:51.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:47:51.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ trash_restore cluster2 mirror test 10b9c75d983f journal 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10b9c75d983f 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:47:51.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 trash restore mirror/10b9c75d983f 2026-02-22T21:47:52.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:47:52.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:47:53.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:47:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:47:53.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:47:53.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:53.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:47:53.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:53.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:53.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:53.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:53.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:53.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:53.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:53.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:53.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:53.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:47:53.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:53.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:53.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:47:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:53.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:47:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:47:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:53.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:53.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:47:53.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:53.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:53.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:47:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:53.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:53.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:53.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:47:53.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:47:53.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:53.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:47:53.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:53.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:47:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:54.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:54.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:54.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:47:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:47:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:47:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:47:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:54.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:47:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:47:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:47:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:47:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:47:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:47:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:47:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:47:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:47:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:47:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:47:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:47:54.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:47:54.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:47:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:47:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:47:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:47:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:54.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T21:47:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:47:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:47:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:47:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:47:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:47:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:47:56.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:47:56.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:47:56.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:47:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:47:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:47:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:47:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:47:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:47:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:47:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed (with rbd-mirror on one cluster)' 2026-02-22T21:47:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:47:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:47:56 TEST: check if removed 'images'\''' OMAP are removed '(with' rbd-mirror on one 'cluster)' 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:47:56 TEST: check if removed images' OMAP are removed (with rbd-mirror on one cluster) 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:56.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:56.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:56.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-22T21:47:56.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-22T21:47:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:47:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-22T21:47:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-22T21:47:56.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T21:47:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=status_global 2026-02-22T21:47:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:56.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-22T21:47:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:56.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=status_global 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c status_global 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:56.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T21:47:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T21:47:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:47:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T21:47:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:56.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_ 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_ 2026-02-22T21:47:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T21:47:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_ 2026-02-22T21:47:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T21:47:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T21:47:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:47:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:56.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_map 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T21:47:56.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T21:47:56.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T21:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 1 = 0 2026-02-22T21:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=1 2026-02-22T21:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T21:47:56.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:56.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:47:57.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T21:47:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:57.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T21:47:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T21:47:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T21:47:57.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T21:47:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 1 = 0 2026-02-22T21:47:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=1 2026-02-22T21:47:57.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:57.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T21:47:57.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:57.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:47:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T21:47:59.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:59.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T21:47:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T21:47:59.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=status_global 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:59.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=status_global 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:59.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T21:47:59.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c status_global 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_ 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_ 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T21:47:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_ 2026-02-22T21:47:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T21:47:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:47:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:59.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_map 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:47:59.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T21:47:59.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T21:47:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error getting omap key set mirror/rbd_mirror_leader: (2) No such file or directory 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test journal 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:47:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:47:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test 2026-02-22T21:47:59.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:47:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:48:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:00.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:00.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:00.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:00.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:00.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:00.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:00.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:48:00.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:00.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:48:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:48:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:48:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:48:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:48:00.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:48:00.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:48:00.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:48:00.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:48:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:00.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:48:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:48:00.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:00.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:48:00.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:48:00.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:48:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:48:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:48:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:48:00.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:48:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:01.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:48:01.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:01.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:48:01.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:48:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:48:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:48:01.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:48:01.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:48:01.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:01.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:01.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:48:02.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:02.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:02.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:02.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:02.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:02.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:02.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:02.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:02.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:02.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:02.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T21:48:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:48:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:48:02.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:48:02.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:48:02.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:48:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:48:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:48:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:48:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:48:02.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T21:48:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:02.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:02.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:02.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:48:02.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:48:02.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:48:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:48:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:48:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:48:02.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T21:48:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:02.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:02.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:02.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:48:02.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:48:02.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:48:02.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:48:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:48:02.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T21:48:02.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T21:48:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:48:02.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:48:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:02.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T21:48:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:04.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:04.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:04.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:04.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:04.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:04.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test 100 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:48:04.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:48:04.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:48:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:48:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.47 bytes/sec: 30 MiB/s 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:04.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:48:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:48:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:48:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:48:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:04.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:04.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:04.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:04.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:04.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:04.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:04.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:04.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:04.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:04.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:04.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:04.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:04.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:04.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:04.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T21:48:04.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:04.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T21:48:04.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:04.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:04.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:04.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:04.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:04.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:04.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:04.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T21:48:04.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T21:48:04.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:48:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:04.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=2 2026-02-22T21:48:04.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:04.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:48:04.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=1 2026-02-22T21:48:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T21:48:04.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:48:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:04.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T21:48:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:48:04.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=3 2026-02-22T21:48:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T21:48:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:48:04.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:04.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:48:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:04.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:04.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:04.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:04.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:04.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:04.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:04.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:04.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:05.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:05.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:05.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:05.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:05.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T21:48:05.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:05.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:05.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T21:48:05.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T21:48:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:05.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:05.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:05.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:05.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:05.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:05.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:05.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:05.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:05.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:05.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:05.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:48:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T21:48:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:03' 2026-02-22T21:48:06.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:48:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:48:06.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T21:48:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:48:06.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:48:06.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:06.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:03' 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:48:03' 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:03' 2026-02-22T21:48:06.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:06.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:06.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43184322mirror.0trial1042026-02-22 21:48:03' 2026-02-22T21:48:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:06.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:03' 2026-02-22T21:48:06.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:48:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:48:06.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:48:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:48:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:06.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: failover and failback' 2026-02-22T21:48:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:48:06.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:48:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:48:06 TEST: failover and failback 2026-02-22T21:48:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:48:06 TEST: failover and failback 2026-02-22T21:48:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster2 2026-02-22T21:48:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:06.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:0 2026-02-22T21:48:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:48:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:48:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:48:06.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:48:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:06.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:1 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:48:06.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T21:48:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:48:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:48:06.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:2 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:48:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T21:48:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T21:48:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:48:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T21:48:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:48:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:48:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:48:06.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:06.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:3 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:48:06.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T21:48:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:48:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:48:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T21:48:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:48:06.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:06.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:48:06.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:48:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:48:06.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:06.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:07.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:07.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:07.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:07.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:07.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:07.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:07.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:48:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:09.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:09.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:09.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:09.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:09.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:09.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:09.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:09.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:09.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:09.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:09.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:09.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:48:09.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:09.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:10.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:10.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:48:10.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:08 2026-02-22T21:48:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:08' 2026-02-22T21:48:10.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:48:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:12.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:12.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:13.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:11 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:11' 2026-02-22T21:48:13.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:13.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:13.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:13.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:13.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:13.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:13.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:11' 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:48:11' 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:11' 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:13.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:48:11cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:48:11' 2026-02-22T21:48:13.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:13.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:11' 2026-02-22T21:48:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:13.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:13.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:13.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:13.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:48:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:14.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:48:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:14.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:11 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:11' 2026-02-22T21:48:14.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:14.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:14.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:14.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:14.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:14.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:14.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:11' 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:48:11' 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:11' 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:14.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:48:11cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:48:11' 2026-02-22T21:48:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:14.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:14.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:11' 2026-02-22T21:48:14.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:14.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:14.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:48:14.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:48:14.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:48:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:48:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:14.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:48:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:48:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:14.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:15.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:48:15.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:15.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:15.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:15.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:15.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:15.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:15.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:15.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:15.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:48:15.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:48:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:17.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:17.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:17.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:17.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:17.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:17.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:17.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:17.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:17.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test 100 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:48:17.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:48:17.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:48:17.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:48:17.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:48:17.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:48:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:48:17.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.47 bytes/sec: 30 MiB/s 2026-02-22T21:48:17.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:17.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:48:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:48:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:48:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:17.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:48:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:17.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:17.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:17.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:17.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:17.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:17.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:17.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:17.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=5, entry_tid=0], [object_number=1, tag_tid=2, entry_tid=101], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:17.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:17.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:17.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:17.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:17.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:17.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:17.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=5, entry_tid=0], [object_number=1, tag_tid=2, entry_tid=101], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:17.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=5, entry_tid=0' '!=' '' 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=5, entry_tid=0' = '' 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:48:17.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:17.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=6 2026-02-22T21:48:17.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:17.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:48:17.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:17.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=5 2026-02-22T21:48:17.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:17.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:48:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T21:48:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:48:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:48:17.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=0 2026-02-22T21:48:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 6 = 5 -a 101 -le 0 2026-02-22T21:48:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:48:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:17.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:48:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:17.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:17.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:17.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=5, entry_tid=0], [object_number=1, tag_tid=2, entry_tid=101], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:17.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:17.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:17.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:17.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:18.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=5, entry_tid=0], [object_number=1, tag_tid=2, entry_tid=101], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:18.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=5, entry_tid=0' '!=' 'object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:18.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:18.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:18.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:18.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:18.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:18.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:18.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:18.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:18.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:18.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:18.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:18.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=5, entry_tid=0], [object_number=1, tag_tid=2, entry_tid=101], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:18.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:18.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:18.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:18.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=5, entry_tid=0], [object_number=1, tag_tid=2, entry_tid=101], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T21:48:18.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:18.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=5, entry_tid=0' '!=' 'object_number=0, tag_tid=5, entry_tid=0' 2026-02-22T21:48:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:18.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.8 2026-02-22T21:48:19.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:19.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:19.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:19.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:19.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:19.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:19.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:19.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:19.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:19.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:19.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:19.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:19.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:19.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:19.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:19.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:19.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:19.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:19.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=6, entry_tid=101], [object_number=0, tag_tid=6, entry_tid=100], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:19.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:19.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:19.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=6, entry_tid=101' -a 'object_number=1, tag_tid=6, entry_tid=101' = 'object_number=1, tag_tid=6, entry_tid=101' 2026-02-22T21:48:19.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:19.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:19.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:20.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:20.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:16 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:48:20.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:48:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":5}} 2026-02-22T21:48:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:16' 2026-02-22T21:48:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:48:20.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:48:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:48:20.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:48:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:48:20.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:20.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:16' 2026-02-22T21:48:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:48:16' 2026-02-22T21:48:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:48:20.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:16' 2026-02-22T21:48:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:20.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:48:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":5}}2026-02-22 21:48:16' 2026-02-22T21:48:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:20.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:16' 2026-02-22T21:48:20.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:20.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:48:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:48:20.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:48:20.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:48:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T21:48:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:20.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:20.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T21:48:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:48:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":5}} 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:16 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:16' 2026-02-22T21:48:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:48:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:48:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:48:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":5}} 2026-02-22T21:48:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:48:21.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:48:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:21.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:21.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:16' 2026-02-22T21:48:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:48:16' 2026-02-22T21:48:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:21.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:16' 2026-02-22T21:48:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:21.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:21.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":5}}43184322mirror.0trial1042026-02-22 21:48:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:48:16' 2026-02-22T21:48:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:21.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:21.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:16' 2026-02-22T21:48:21.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:21.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:48:21.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:48:21.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:48:21.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T21:48:21.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:21.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:48:21.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T21:48:21.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T21:48:21.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:21.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:48:21.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:48:21.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:21.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:48:21.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:21.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:21.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:48:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:21.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:48:21.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:48:21.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:48:21.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:21.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:21.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:21.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:22.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:22.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:22.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:22.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:22.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:22.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:22.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:22.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:22.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:22.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:24.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:24.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:24.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:24.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:24.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:24.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:24.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:24.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:24.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:24.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:24.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:24.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:24.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:24.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:25.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:25.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:25.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:25.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:25.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:25.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:48:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:48:25.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:25.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:26.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:26.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:23 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:21' 2026-02-22T21:48:26.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:26.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:48:26.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:26.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:28.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:28.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:26 2026-02-22T21:48:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:26' 2026-02-22T21:48:28.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:28.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:28.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:28.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:26' 2026-02-22T21:48:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:48:26' 2026-02-22T21:48:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:26' 2026-02-22T21:48:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:48:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:48:26' 2026-02-22T21:48:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:28.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:28.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:26' 2026-02-22T21:48:28.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:28.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:28.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:28.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:48:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:28.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:28.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:29.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:29.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:48:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:26 2026-02-22T21:48:29.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:48:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:26' 2026-02-22T21:48:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:29.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:29.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:29.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:29.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:29.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:26' 2026-02-22T21:48:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:48:26' 2026-02-22T21:48:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:29.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:26' 2026-02-22T21:48:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:48:26cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:48:26' 2026-02-22T21:48:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:29.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:29.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:26' 2026-02-22T21:48:29.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:29.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:29.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:48:29.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:29.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:29.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:29.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:48:29.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:48:29.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:48:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:48:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:29.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:48:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:30.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:30.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:30.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:48:30.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:48:30.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:30.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:30.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:48:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:30.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:30.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:48:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:30.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:48:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:32.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:32.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:48:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:48:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:32.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:48:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:32.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:48:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:48:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:48:32.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:32.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:32.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:48:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:48:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:32.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:48:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:32.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:32.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:32.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:32.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:48:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:48:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:32.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:48:32.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:48:32.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:48:32.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:32.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:32.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:48:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:33.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:48:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:48:33.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:48:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:48:33.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:48:33.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:48:33.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:48:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:48:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:48:33.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:48:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:48:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:33.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:48:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:48:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:33.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:48:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:33.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:33.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:33.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:48:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:33.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:35.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:48:35.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:48:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:35.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:35.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:48:35.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:35.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:35.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:35.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:48:35.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:48:35.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:35.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:35.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:48:35.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:35.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:35.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:35.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:35.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:35.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:35.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:35.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:36.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:36.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:36.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:36.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:36.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:48:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:36.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:31 2026-02-22T21:48:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:48:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:33' 2026-02-22T21:48:36.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:48:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:36.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:36 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:36' 2026-02-22T21:48:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:38.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:38.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:36' 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:48:36' 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:36' 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:38.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:48:36cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:48:36' 2026-02-22T21:48:38.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:38.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:36' 2026-02-22T21:48:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:38.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:38.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:48:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:38.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:38.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:48:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:39.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:39.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:39.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:39.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:48:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:39.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:36 2026-02-22T21:48:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:48:39.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:36' 2026-02-22T21:48:39.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:39.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:39.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:39.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:39.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:39.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:36' 2026-02-22T21:48:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:48:36' 2026-02-22T21:48:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:39.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:36' 2026-02-22T21:48:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:39.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:39.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:39.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:48:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:48:36' 2026-02-22T21:48:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:39.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:36' 2026-02-22T21:48:39.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:39.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:39.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:39.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:39.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:39.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:48:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:48:39.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:48:39.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:48:39.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:48:39.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:39.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:48:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:40.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:40.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:40.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:40.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:40.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:40.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:40.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:40.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:40.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:40.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:40.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:40.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:48:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:40.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:48:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:42.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:42.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:42.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:42.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:42.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:42.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:42.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:42.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:42.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:42.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:42.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:42.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:42.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:42.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:42.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:48:42.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:48:43.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:48:43.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:48:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:48:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:48:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:43.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:48:43.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:43.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:43.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:48:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:48:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:43.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:43.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:48:43.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:48:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:48:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:43.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:48:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:43.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:43.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=11, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:43.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=11, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:43.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:43.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=11, entry_tid=0' 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:48:43.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:48:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:48:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:48:43.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:43.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:48:43.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:48:43.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:48:43.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=11, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:43.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=11, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:48:43.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:48:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=11, entry_tid=0' 2026-02-22T21:48:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=11, entry_tid=0' -a 'object_number=0, tag_tid=11, entry_tid=0' = 'object_number=0, tag_tid=11, entry_tid=0' 2026-02-22T21:48:43.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:48:43.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:43.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T21:48:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:44.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:48:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:44.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:44.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":8},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":11}} 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:41 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:41' 2026-02-22T21:48:44.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:48:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:48:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:48:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":8},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":11}} 2026-02-22T21:48:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:48:44.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:48:44.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:44.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:41' 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:48:41' 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:41' 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:44.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":8},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":11}}43184322mirror.0trial1042026-02-22 21:48:41cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:48:41' 2026-02-22T21:48:44.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:41' 2026-02-22T21:48:44.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:44.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:48:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:48:44.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:44.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:48:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:48:45.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:48:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:48:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:41 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":8},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":11}} 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:41' 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:48:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:48:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:48:45.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:48:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:41' 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:48:41' 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:41' 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:48:41cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":8},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":11}}2026-02-22 21:48:41' 2026-02-22T21:48:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:45.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:41' 2026-02-22T21:48:45.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:45.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:48:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:48:45.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:48:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:48:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:45.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T21:48:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:48:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T21:48:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T21:48:45.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:48:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T21:48:45.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T21:48:45.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:48:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:45.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:45.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:48:45.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:48:45.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:45.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:48:45.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:48:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:48:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:45.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:48:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:48:45.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:48:45.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:48:45.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:45.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:46.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:46.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:46.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:46.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:46.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:46.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:46.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:46.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:46.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:46.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:48:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:48:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:46.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:48:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:48:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:48:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:46.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:48:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:48:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:48:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:48:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:48:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:48:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:48:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:48:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:48:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:48.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:48:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:48:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:48.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:48.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:48:48.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:48.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:48:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:49.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:49.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:48:49.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:47 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:46' 2026-02-22T21:48:49.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:48:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:49.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:51.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:48:51.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:51.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:51.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:51.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:51 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:51' 2026-02-22T21:48:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:51.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:51.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:51' 2026-02-22T21:48:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:48:51' 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:51' 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:48:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:48:51' 2026-02-22T21:48:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:51.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:51.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:51' 2026-02-22T21:48:51.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:51.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:51.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:51.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:51.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:51.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:48:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:52.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:48:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:48:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:52.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:48:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:48:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:48:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:48:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:51 2026-02-22T21:48:53.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:48:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:48:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:48:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:48:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:51' 2026-02-22T21:48:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:48:53.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:48:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:48:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:48:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:48:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:48:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:48:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:48:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:48:53.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:48:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:51' 2026-02-22T21:48:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:48:51' 2026-02-22T21:48:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:48:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:51' 2026-02-22T21:48:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:48:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:48:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:48:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:48:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:48:51' 2026-02-22T21:48:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:48:53.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:48:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:51' 2026-02-22T21:48:53.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:48:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:48:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:48:53.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:48:53.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:48:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:48:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:48:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:48:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:53.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:48:53.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:48:53.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:48:53.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:48:53.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:53.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:48:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:48:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:54.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:48:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:48:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:54.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:48:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:48:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:48:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:48:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:48:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:48:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:48:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:48:56.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:48:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:48:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:48:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:48:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:48:56.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:48:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:48:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:48:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:48:56.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:48:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:48:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:48:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:48:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:48:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:48:56.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:48:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:48:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:48:56.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:48:56.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:56.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:56.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:48:56.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:48:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:48:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:48:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:48:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:48:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:48:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:48:56.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:49:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:49:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:00.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:00.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:00.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:00.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:00.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:00.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:00.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:00.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:00.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:00.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:00.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:49:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:49:00.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:00.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:49:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:49:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:49:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster1 mirror test 100 2026-02-22T21:49:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:00.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:49:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:49:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:49:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:49:00.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:49:00.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:49:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:49:00.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 10000.2 bytes/sec: 39 MiB/s 2026-02-22T21:49:00.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:00.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:49:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:49:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:49:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:00.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:00.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:00.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:00.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:00.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:00.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:49:00.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:49:00.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:00.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:00.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:49:00.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:49:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:00.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:00.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:00.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:00.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:00.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:00.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:00.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:49:00.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:00.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:00.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:00.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:49:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:00.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:00.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:00.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=12, entry_tid=0]]], state=connected] 2026-02-22T21:49:00.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:00.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:00.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=12, entry_tid=0]]], state=connected] 2026-02-22T21:49:00.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:00.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos= 2026-02-22T21:49:00.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=13, entry_tid=101' -a 'object_number=1, tag_tid=13, entry_tid=101' = '' 2026-02-22T21:49:00.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test '' '!=' '' 2026-02-22T21:49:00.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:00.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T21:49:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:49:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:49:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:01.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:49:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:49:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:01.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:01.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:01.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:01.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:01.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=12, entry_tid=0]]], state=connected] 2026-02-22T21:49:01.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:01.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:01.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:01.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:01.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=12, entry_tid=0]]], state=connected] 2026-02-22T21:49:01.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos= 2026-02-22T21:49:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=13, entry_tid=101' -a 'object_number=1, tag_tid=13, entry_tid=101' = '' 2026-02-22T21:49:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test '' '!=' '' 2026-02-22T21:49:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:01.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.8 2026-02-22T21:49:02.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:02.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:49:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:02.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:49:02.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:02.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:02.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:02.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:49:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:02.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:02.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:02.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:02.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:02.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:02.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T21:49:02.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:02.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:02.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:02.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=1, tag_tid=13, entry_tid=101], [object_number=0, tag_tid=13, entry_tid=100], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:02.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T21:49:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=13, entry_tid=101' -a 'object_number=1, tag_tid=13, entry_tid=101' = 'object_number=1, tag_tid=13, entry_tid=101' 2026-02-22T21:49:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:49:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:02.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:02.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:01 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:03.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":9},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":12}} 2026-02-22T21:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:56' 2026-02-22T21:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:49:03.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:49:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:03.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:03.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:03.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:01' 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:49:01' 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:01' 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:03.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:49:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":9},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":12}}2026-02-22 21:48:56' 2026-02-22T21:49:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:03.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:03.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:01' 2026-02-22T21:49:03.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:03.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:49:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:49:03.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:03.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror test 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying primary_position 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:03.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:03.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying primary_position '' 2026-02-22T21:49:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:04.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:49:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":9},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":12}} 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:48:56 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:01' 2026-02-22T21:49:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:49:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:49:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":9},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":12}} 2026-02-22T21:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:04.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:04.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:56' 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:48:56' 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:48:56' 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":9},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":12}}45054508mirror.0trial1042026-02-22 21:48:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:49:01' 2026-02-22T21:49:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:48:56' 2026-02-22T21:49:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:49:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:04.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:49:04.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:04.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T21:49:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:49:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T21:49:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T21:49:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:04.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:49:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T21:49:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T21:49:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:04.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:04.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:49:04.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:49:04.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:04.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:49:04.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:04.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:04.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:49:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:04.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:49:04.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:49:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:49:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:49:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:04.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:49:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:04.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:49:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:49:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:05.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:05.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:05.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:05.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:05.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:05.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:05.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:05.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:49:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:49:05.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:49:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:49:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:49:05.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:05.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:49:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:49:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:07.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:07.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:07.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:07.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:07.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:07.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:07.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:07.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:49:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:49:07.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:07.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:08.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:06 2026-02-22T21:49:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:06' 2026-02-22T21:49:08.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:08.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:08.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:08.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:08.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:06' 2026-02-22T21:49:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:49:06' 2026-02-22T21:49:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:08.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:06' 2026-02-22T21:49:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:08.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:49:06cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:49:06' 2026-02-22T21:49:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:09.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:09.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:06' 2026-02-22T21:49:09.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:09.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:09.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:09.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:49:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:06 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:06' 2026-02-22T21:49:10.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:10.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:10.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:10.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:10.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:06' 2026-02-22T21:49:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:49:06' 2026-02-22T21:49:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:10.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:06' 2026-02-22T21:49:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:10.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:49:06cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:49:06' 2026-02-22T21:49:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:10.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:06' 2026-02-22T21:49:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:10.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:10.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:49:10.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:49:10.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:49:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:49:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:10.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:10.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:49:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:11.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:11.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:11.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:11.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:11.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:11.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:11.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:11.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:11.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:11.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:11.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:11.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:49:11.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:11.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:49:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:13.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:13.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:13.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:13.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:13.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:13.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:49:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:49:13.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:13.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:49:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:49:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:49:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test 100 2026-02-22T21:49:13.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:49:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:49:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:49:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:49:13.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:49:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:49:13.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:49:13.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7143.01 bytes/sec: 28 MiB/s 2026-02-22T21:49:13.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:49:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:49:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:49:13.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:49:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:49:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:49:13.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:49:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:49:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:13.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:49:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:49:13.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:49:13.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:13.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:49:13.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:13.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:13.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:13.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:49:13.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:13.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:13.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:13.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:13.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:13.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:13.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=17, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:49:13.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:49:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:49:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:13.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:13.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:13.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:13.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:13.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=17, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:49:13.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:13.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=18, entry_tid=101' -a 'object_number=1, tag_tid=18, entry_tid=101' = 'object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=17, entry_tid=0' '!=' '' 2026-02-22T21:49:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:49:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=17, entry_tid=0' = '' 2026-02-22T21:49:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:13.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:49:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:49:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=18 2026-02-22T21:49:13.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:49:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:49:13.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=17 2026-02-22T21:49:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:49:13.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:49:13.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T21:49:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:49:13.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:49:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=0 2026-02-22T21:49:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 18 = 17 -a 101 -le 0 2026-02-22T21:49:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:49:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:13.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:49:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:49:13.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:49:13.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:13.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:49:13.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:13.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:13.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:13.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:13.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:49:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:49:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:13.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:13.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:13.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=17, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:49:13.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:49:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:49:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:13.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:13.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:13.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:14.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:14.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=17, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:49:14.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=18, entry_tid=101' -a 'object_number=1, tag_tid=18, entry_tid=101' = 'object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=17, entry_tid=0' '!=' 'object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:14.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:14.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:14.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:14.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:14.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:14.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:14.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:14.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:14.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=17, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:49:14.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:14.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:14.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:14.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=17, entry_tid=0], [object_number=1, tag_tid=6, entry_tid=101], [object_number=3, tag_tid=6, entry_tid=99], [object_number=2, tag_tid=6, entry_tid=98]]], state=connected] 2026-02-22T21:49:14.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:14.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=18, entry_tid=101' -a 'object_number=1, tag_tid=18, entry_tid=101' = 'object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=0, tag_tid=17, entry_tid=0' '!=' 'object_number=0, tag_tid=17, entry_tid=0' 2026-02-22T21:49:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:14.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.8 2026-02-22T21:49:15.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:49:15.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:49:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:49:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:15.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:49:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:15.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:49:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:49:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:15.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:15.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:15.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:15.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:15.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:15.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:15.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:15.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:15.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:15.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:15.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:15.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:15.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:15.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:15.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:15.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:15.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=18, entry_tid=101], [object_number=0, tag_tid=18, entry_tid=100], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:15.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=18, entry_tid=101' -a 'object_number=1, tag_tid=18, entry_tid=101' = 'object_number=1, tag_tid=18, entry_tid=101' 2026-02-22T21:49:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T21:49:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:15.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:15.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:16.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:16.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:16.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":13},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":17}} 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:11 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:11' 2026-02-22T21:49:16.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:49:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:49:16.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:49:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":13},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":17}} 2026-02-22T21:49:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:16.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:16.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:16.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:16.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:16.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:16.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:16.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:11' 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:49:11' 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:11' 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:16.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":13},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":17}}43184322mirror.0trial1042026-02-22 21:49:11cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:49:11' 2026-02-22T21:49:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:16.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:16.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:11' 2026-02-22T21:49:16.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:49:16.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:16.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:49:16.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:49:16.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:49:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:17.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:17.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:17.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:17.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:16 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":18},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":18}} 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:16' 2026-02-22T21:49:17.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:49:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:49:17.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:17.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:49:17.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:17.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:17.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:17.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:17.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:17.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:17.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:16' 2026-02-22T21:49:17.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:49:16' 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:16' 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:17.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:49:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":18},"primary_position":{"entry_tid":101,"object_number":1,"tag_tid":18}}2026-02-22 21:49:16' 2026-02-22T21:49:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:17.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:16' 2026-02-22T21:49:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:17.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:49:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:49:17.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:17.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T21:49:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:49:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T21:49:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T21:49:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:17.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T21:49:17.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:49:17.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:49:17.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:18.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T21:49:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: failover / failback loop' 2026-02-22T21:49:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:49:18.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:49:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:49:18 TEST: failover / failback loop 2026-02-22T21:49:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:49:18 TEST: failover / failback loop 2026-02-22T21:49:18.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 1 20 2026-02-22T21:49:18.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:49:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:49:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:18.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:49:18.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:49:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:49:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:49:18.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:49:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:18.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:49:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:49:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:19.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:19.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:19.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:19.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:19.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:19.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:19.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:19.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:19.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:19.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:19.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:49:19.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:19.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:20.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:20.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:19 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:20.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:16' 2026-02-22T21:49:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:49:20.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:20.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:49:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:22.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:22.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:21 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:22.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:21' 2026-02-22T21:49:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:22.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:22.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:22.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:22.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:21' 2026-02-22T21:49:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:49:21' 2026-02-22T21:49:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:22.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:21' 2026-02-22T21:49:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:22.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:22.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:49:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:49:21' 2026-02-22T21:49:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:22.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:21' 2026-02-22T21:49:22.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:22.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:22.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:22.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:22.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:23.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:23.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:23.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:21 2026-02-22T21:49:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:21' 2026-02-22T21:49:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:23.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:23.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:23.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:23.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:21' 2026-02-22T21:49:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:49:21' 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:21' 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:49:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:49:21' 2026-02-22T21:49:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:23.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:23.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:21' 2026-02-22T21:49:23.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:49:23.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:49:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:49:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:49:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:49:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:24.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:24.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:24.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:24.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:24.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:24.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:24.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:24.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:24.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:24.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:24.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:49:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:24.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:49:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:26.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:26.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:26.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:26.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:26.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:26.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:26.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:26.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:26.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:49:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:26.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:49:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:49:30.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:30.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:30.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:30.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:30.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:30.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:30.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:30.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:30.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:30.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:30.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:30.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:30.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:30.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:30.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:49:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:49:30.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:30.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:30.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:30.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:49:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:49:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:30.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:49:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=19, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:30.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=19, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:30.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:30.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=19, entry_tid=0' 2026-02-22T21:49:30.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:49:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=19, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:30.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=19, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:49:30.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=19, entry_tid=0' 2026-02-22T21:49:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=19, entry_tid=0' -a 'object_number=0, tag_tid=19, entry_tid=0' = 'object_number=0, tag_tid=19, entry_tid=0' 2026-02-22T21:49:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:49:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:30.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:49:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:31.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:31 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":15},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":19}} 2026-02-22T21:49:31.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:26' 2026-02-22T21:49:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:49:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:49:31.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:49:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:31.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:31.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:31.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:31' 2026-02-22T21:49:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:49:31' 2026-02-22T21:49:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:31.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:31' 2026-02-22T21:49:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:31.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:31.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:49:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":15},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":19}}2026-02-22 21:49:26' 2026-02-22T21:49:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:31.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:31.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:31' 2026-02-22T21:49:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:49:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:31.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:49:31.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:31.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:31.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:49:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:32.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:33.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":15},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":19}} 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:26 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:31' 2026-02-22T21:49:33.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:49:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:49:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:33.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":15},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":19}} 2026-02-22T21:49:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:33.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:33.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:33.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:26' 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:49:26' 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:26' 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:33.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":15},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":19}}45054508mirror.0trial1042026-02-22 21:49:26cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:49:31' 2026-02-22T21:49:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:33.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:26' 2026-02-22T21:49:33.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:33.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:49:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:49:33.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:33.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:49:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:33.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:49:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:49:33.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:33.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:34.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:34.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:34.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:34.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:34.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:49:34.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:49:34.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:34.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:49:34.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:49:34.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:49:34.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:34.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:49:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:36.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:36.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:36.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:36.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:36.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:36.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:36.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:36.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:36.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:49:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:49:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:36.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:36.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:49:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:37.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:36 2026-02-22T21:49:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:36' 2026-02-22T21:49:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:36' 2026-02-22T21:49:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:49:36' 2026-02-22T21:49:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:37.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:36' 2026-02-22T21:49:37.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:37.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:37.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:49:36cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:49:36' 2026-02-22T21:49:37.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:37.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:36' 2026-02-22T21:49:37.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:37.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:37.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:37.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:37.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:38.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:36 2026-02-22T21:49:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:36' 2026-02-22T21:49:38.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:36' 2026-02-22T21:49:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:49:36' 2026-02-22T21:49:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:36' 2026-02-22T21:49:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:38.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:49:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:49:36' 2026-02-22T21:49:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:38.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:36' 2026-02-22T21:49:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:38.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:38.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:38.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:49:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:49:38.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:49:38.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:49:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:49:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:38.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:49:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:49:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:38.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:38.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:49:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:39.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:39.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:39.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:39.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:39.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:39.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:39.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:39.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:39.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:39.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:39.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:49:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:39.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:41.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:49:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:41.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:41.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:41.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:41.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:41.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:41.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:41.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:49:41.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:41.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:49:45.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:49:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:45.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:45.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:45.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:45.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:45.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:45.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:45.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:45.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:45.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:49:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:49:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:45.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:49:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:49:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:49:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:45.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:49:45.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:49:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:49:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:49:45.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:49:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:49:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:49:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:49:45.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:49:45.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:45.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:49:45.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:45.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:45.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:45.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:45.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:49:45.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:49:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:49:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:45.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:49:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:45.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:45.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:45.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=23, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:45.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=23, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:45.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=23, entry_tid=0' 2026-02-22T21:49:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:49:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:45.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:49:45.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:49:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:49:45.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:45.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:49:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:49:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:49:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=23, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=23, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:49:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:49:45.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=23, entry_tid=0' 2026-02-22T21:49:45.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=23, entry_tid=0' -a 'object_number=0, tag_tid=23, entry_tid=0' = 'object_number=0, tag_tid=23, entry_tid=0' 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:45.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:46.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:46.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:46 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23}} 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:46' 2026-02-22T21:49:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:49:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:49:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:46.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:49:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:46.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:46.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:46' 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:49:46' 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:46' 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:46.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:47.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:49:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23}}2026-02-22 21:49:46' 2026-02-22T21:49:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:47.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:46' 2026-02-22T21:49:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:49:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:49:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:49:47.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:47.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:49:48.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:48.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:49:48.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23}} 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:46 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:46' 2026-02-22T21:49:48.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:49:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:49:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:48.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23}} 2026-02-22T21:49:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:48.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:48.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:46' 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:49:46' 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:46' 2026-02-22T21:49:48.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:48.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":23}}43184322mirror.0trial1042026-02-22 21:49:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:49:46' 2026-02-22T21:49:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:48.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:46' 2026-02-22T21:49:48.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:48.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:49:48.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:49:48.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:49:48.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:48.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:49:48.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:49:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:49:48.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:48.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:49.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:49.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:49.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:49.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:49.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:49.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:49:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:49:49.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:49:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:49:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:49:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:49:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:49:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:51.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:49:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:49:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:49:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:49:51.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:49:51.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:51.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:49:51.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:49:51.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:51.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:49:51.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:49:51.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:51.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:51.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:51.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:51.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:52.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:52.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:49:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:52.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:51 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:51' 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:52.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:52.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:52.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:52.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:52.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:51' 2026-02-22T21:49:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:49:51' 2026-02-22T21:49:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:52.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:51' 2026-02-22T21:49:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:52.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:49:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:49:51' 2026-02-22T21:49:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:51' 2026-02-22T21:49:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:52.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:52.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:49:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:52.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:52.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:49:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:53.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:49:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:49:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:53.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:49:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:49:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:49:53.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:51 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:51' 2026-02-22T21:49:53.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:49:53.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:49:53.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:49:53.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:49:53.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:49:53.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:53.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:49:53.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:53.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:49:53.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:49:53.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:49:53.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:49:53.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:51' 2026-02-22T21:49:53.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:49:51' 2026-02-22T21:49:53.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:49:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:51' 2026-02-22T21:49:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:49:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:49:53.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:49:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:49:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:49:51' 2026-02-22T21:49:53.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:49:53.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:49:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:51' 2026-02-22T21:49:53.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:49:53.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:49:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:49:53.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:49:53.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:49:53.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:53.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:49:53.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:49:53.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:49:53.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:53.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:53.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:49:53.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:49:53.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:49:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:49:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:49:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:53.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:49:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:49:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:53.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:49:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:49:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:54.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:54.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:49:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:49:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:49:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:49:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:49:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:49:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:56.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:49:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:49:56.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:49:56.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:49:56.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:49:56.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:49:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:49:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:49:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:49:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:49:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:49:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:49:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:49:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:49:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:50:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:50:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:00.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:00.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:00.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:00.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:00.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:00.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:00.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:00.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:00.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:00.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:00.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:00.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:00.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:50:00.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:50:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:50:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:00.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:50:00.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:00.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:50:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:00.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:50:00.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:00.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:00.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:50:00.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:00.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:00.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:50:00.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=24, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:01.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=24, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:01.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:01.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=24, entry_tid=0' 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:50:01.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:01.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:50:01.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:01.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=24, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:01.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=24, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:01.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=24, entry_tid=0' 2026-02-22T21:50:01.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=24, entry_tid=0' -a 'object_number=0, tag_tid=24, entry_tid=0' = 'object_number=0, tag_tid=24, entry_tid=0' 2026-02-22T21:50:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:50:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:01.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:01.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:50:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:02.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:02.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:02.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:01 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:02.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":21},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":24}} 2026-02-22T21:50:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:56' 2026-02-22T21:50:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:50:02.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:50:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:02.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:50:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:02.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:02.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:01' 2026-02-22T21:50:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:50:01' 2026-02-22T21:50:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:02.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:01' 2026-02-22T21:50:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:02.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:02.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:50:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":21},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":24}}2026-02-22 21:49:56' 2026-02-22T21:50:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:02.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:01' 2026-02-22T21:50:02.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:02.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:50:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:50:02.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:02.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:02.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":21},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":24}} 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:49:56 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:01' 2026-02-22T21:50:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:50:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:50:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":21},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":24}} 2026-02-22T21:50:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:03.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:56' 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:49:56' 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:49:56' 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":21},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":24}}45054508mirror.0trial1042026-02-22 21:49:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:50:01' 2026-02-22T21:50:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:03.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:49:56' 2026-02-22T21:50:03.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:50:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:03.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:03.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:50:03.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:50:03.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:50:03.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:03.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:03.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:03.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:50:03.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:03.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:03.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:03.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:50:03.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:03.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:03.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:50:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:04.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:04.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:04.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:04.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:04.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:04.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:04.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:04.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:50:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:06.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:50:06.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:06.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:06.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:06.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:06.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:06.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:06.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:06.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:06.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:06.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:06.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:06.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:06.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:50:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:50:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:06.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:06.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:50:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:07.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:07.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:07.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:07.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:07.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:07.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:07.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:07.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:06 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:06' 2026-02-22T21:50:07.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:07.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:07.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:07.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:07.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:06' 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:50:06' 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:06' 2026-02-22T21:50:07.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:07.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:07.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:50:06cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:50:06' 2026-02-22T21:50:07.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:07.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:06' 2026-02-22T21:50:07.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:07.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:07.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:07.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:08.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:08.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:08.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:08.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:06 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:06' 2026-02-22T21:50:08.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:08.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:08.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:08.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:08.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:08.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:08.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:08.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:08.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:08.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:08.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:08.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:06' 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:50:06' 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:06' 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:08.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:08.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:50:06cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:50:06' 2026-02-22T21:50:08.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:08.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:08.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:06' 2026-02-22T21:50:08.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:08.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:08.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:50:08.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:50:08.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:50:08.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:08.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:09.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:50:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:09.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:09.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:09.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:09.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:09.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:09.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:09.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:09.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:09.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:50:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:09.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:50:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:11.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:11.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:11.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:11.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:11.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:11.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:11.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:11.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:11.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:50:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:11.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:50:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:50:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:15.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:15.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:15.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:15.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:15.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:15.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:15.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:15.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:15.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:50:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:50:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:50:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:50:15.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:50:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:50:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:16.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:16.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:50:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:50:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:50:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:16.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:50:16.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:50:16.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:16.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:16.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:16.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:50:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:16.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:50:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:50:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:16.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:50:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:50:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=28, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:16.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=28, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:16.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:16.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=28, entry_tid=0' 2026-02-22T21:50:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:50:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:16.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:16.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:50:16.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=28, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:16.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=28, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:16.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=28, entry_tid=0' 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=28, entry_tid=0' -a 'object_number=0, tag_tid=28, entry_tid=0' = 'object_number=0, tag_tid=28, entry_tid=0' 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:16.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:50:17.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:17.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:17.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:16 2026-02-22T21:50:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":4.8,"entries_behind_primary":0,"entries_per_second":0.2,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28}} 2026-02-22T21:50:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:16' 2026-02-22T21:50:17.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:50:17.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:50:17.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:17.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:50:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:17.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:17.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:17.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:17.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:16' 2026-02-22T21:50:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:50:16' 2026-02-22T21:50:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:17.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:16' 2026-02-22T21:50:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:50:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":4.8,"entries_behind_primary":0,"entries_per_second":0.2,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28}}2026-02-22 21:50:16' 2026-02-22T21:50:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:17.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:16' 2026-02-22T21:50:17.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:50:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:50:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:50:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:17.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:50:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:18.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:18.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":4.8,"entries_behind_primary":0,"entries_per_second":0.2,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28}} 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:16 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:16' 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:50:18.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:50:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":4.8,"entries_behind_primary":0,"entries_per_second":0.2,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28}} 2026-02-22T21:50:18.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:18.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:18.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:18.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:18.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:16' 2026-02-22T21:50:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:50:16' 2026-02-22T21:50:18.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:16' 2026-02-22T21:50:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:18.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:18.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":4.8,"entries_behind_primary":0,"entries_per_second":0.2,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":28}}43184322mirror.0trial1042026-02-22 21:50:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:50:16' 2026-02-22T21:50:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:18.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:16' 2026-02-22T21:50:18.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:18.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:50:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:50:18.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:18.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:50:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:50:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:18.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:50:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:50:18.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:50:18.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:18.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:50:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:18.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:50:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:18.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:19.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:19.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:19.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:19.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:19.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:19.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:19.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:19.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:19.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:19.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:19.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:19.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:19.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:19.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:50:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:19.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:50:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:21.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:21.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:21.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:21.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:21.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:21.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:21.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:21.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:21.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:21.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:21.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:21.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:21.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:50:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:50:22.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:22.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:22.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:22.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:21 2026-02-22T21:50:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:21' 2026-02-22T21:50:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:22.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:22.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:22.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:22.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:22.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:21' 2026-02-22T21:50:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:50:21' 2026-02-22T21:50:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:22.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:21' 2026-02-22T21:50:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:22.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:50:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:50:21' 2026-02-22T21:50:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:22.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:22.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:21' 2026-02-22T21:50:22.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:22.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:22.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:22.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:50:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:22.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:22.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:23.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:23.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:23.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:23.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:21 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:21' 2026-02-22T21:50:23.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:23.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:23.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:21' 2026-02-22T21:50:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:50:21' 2026-02-22T21:50:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:23.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:21' 2026-02-22T21:50:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:23.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:50:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:50:21' 2026-02-22T21:50:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:21' 2026-02-22T21:50:23.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:23.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:50:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:50:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:50:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:50:24.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:24.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:24.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:24.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:24.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:24.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:24.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:24.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:24.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:50:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:27.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:27.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:27.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:27.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:27.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:27.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:27.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:27.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:27.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:27.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:27.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:27.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:50:27.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:50:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:50:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:50:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:50:27.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:50:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:50:27.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:27.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:50:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:50:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:50:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:27.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:50:27.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:27.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:50:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:50:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:27.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:27.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:50:27.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:50:27.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:50:27.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=29, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:27.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=29, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:27.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=29, entry_tid=0' 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:50:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:27.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:27.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:27.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:50:27.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=29, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=29, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:27.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=29, entry_tid=0' 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=29, entry_tid=0' -a 'object_number=0, tag_tid=29, entry_tid=0' = 'object_number=0, tag_tid=29, entry_tid=0' 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:27.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:50:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:28.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:28.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:26 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:28.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":26},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":29}} 2026-02-22T21:50:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:26' 2026-02-22T21:50:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:50:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:50:28.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:50:28.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:28.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:28.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:28.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:28.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:28.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:26' 2026-02-22T21:50:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:50:26' 2026-02-22T21:50:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:28.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:26' 2026-02-22T21:50:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:28.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:28.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:50:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":26},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":29}}2026-02-22 21:50:26' 2026-02-22T21:50:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:28.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:28.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:26' 2026-02-22T21:50:28.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:50:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:28.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:50:28.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:28.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:28.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:28.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:50:28.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:28.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:28.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:28.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:29.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:29.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":26},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":29}} 2026-02-22T21:50:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:29.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:26 2026-02-22T21:50:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:26' 2026-02-22T21:50:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:50:29.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:50:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:29.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":26},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":29}} 2026-02-22T21:50:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:29.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:29.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:29.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:26' 2026-02-22T21:50:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:50:26' 2026-02-22T21:50:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:29.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:26' 2026-02-22T21:50:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:29.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:29.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":26},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":29}}45054508mirror.0trial1042026-02-22 21:50:26cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:50:26' 2026-02-22T21:50:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:29.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:29.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:26' 2026-02-22T21:50:29.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:50:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:29.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:50:29.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:29.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:50:29.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:50:29.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:29.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:50:29.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:29.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:29.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:30.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:30.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:30.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:30.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:30.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:30.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:30.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:30.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:50:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:30.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:50:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:32.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:32.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:32.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:32.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:32.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:32.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:32.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:32.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:32.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:32.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:32.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:32.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:32.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:32.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:32.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:32.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:32.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:50:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:50:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:32.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:32.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:50:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:33.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:33.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:31 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:33.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:31' 2026-02-22T21:50:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:33.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:33.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:33.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:33.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:33.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:33.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:31' 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:50:31' 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:33.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:31' 2026-02-22T21:50:33.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:33.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:33.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:33.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:50:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:50:31' 2026-02-22T21:50:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:33.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:31' 2026-02-22T21:50:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:33.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:34.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:50:34.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:34.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:34.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:34.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:34.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:34.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:31 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:31' 2026-02-22T21:50:34.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:34.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:34.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:34.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:34.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:31' 2026-02-22T21:50:34.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:50:31' 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:31' 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:34.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:50:31cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:50:31' 2026-02-22T21:50:34.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:34.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:31' 2026-02-22T21:50:34.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:34.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:50:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:34.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:50:34.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:50:34.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:50:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:50:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:34.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:34.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:50:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:35.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:35.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:35.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:35.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:35.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:35.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:35.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:35.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:35.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:50:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:37.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:37.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:37.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:37.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:37.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:37.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:37.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:37.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:37.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:38.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:38.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:38.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:50:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:50:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:50:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:50:38.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:50:38.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:50:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:50:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:50:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:50:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:50:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:50:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:50:38.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:38.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:38.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:50:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:50:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:38.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:38.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:38.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:50:38.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:50:38.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:38.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:50:38.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=33, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:38.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=33, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:38.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:38.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=33, entry_tid=0' 2026-02-22T21:50:38.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:38.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:50:38.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=33, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:38.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=33, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:50:38.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=33, entry_tid=0' 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=33, entry_tid=0' -a 'object_number=0, tag_tid=33, entry_tid=0' = 'object_number=0, tag_tid=33, entry_tid=0' 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:38.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:38.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:50:39.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:39.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:36 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":30},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":33}} 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:36' 2026-02-22T21:50:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:50:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:50:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:50:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:39.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:39.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:36' 2026-02-22T21:50:39.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:50:36' 2026-02-22T21:50:39.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:39.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:36' 2026-02-22T21:50:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:39.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:39.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:50:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":30},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":33}}2026-02-22 21:50:36' 2026-02-22T21:50:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:36' 2026-02-22T21:50:39.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:39.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:50:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:50:39.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:50:39.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:39.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:39.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:40.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:40.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":30},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":33}} 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:36 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:40.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:50:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:36' 2026-02-22T21:50:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:50:40.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:50:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:40.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":30},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":33}} 2026-02-22T21:50:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:40.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:40.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:40.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:36' 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:50:36' 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:36' 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:40.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:40.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:40.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":30},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":33}}43184322mirror.0trial1042026-02-22 21:50:36cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:50:36' 2026-02-22T21:50:40.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:40.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:40.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:36' 2026-02-22T21:50:40.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:50:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:50:40.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:50:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:50:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:50:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:50:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:40.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:40.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:40.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:50:40.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:50:40.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:50:40.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:40.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:40.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:50:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:41.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:41.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:41.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:41.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:41.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:41.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:41.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:41.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:41.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:41.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:41.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:41.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:41.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:41.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:41.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:41.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:41.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:41.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:50:41.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:41.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:43.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:50:43.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:43.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:50:43.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:50:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:50:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:43.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:43.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:50:43.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:50:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:43.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:43.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:43.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:43.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:50:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:44.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:44.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:42 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:41' 2026-02-22T21:50:44.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:50:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:44.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:46.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:46.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:46.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:42 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:46' 2026-02-22T21:50:46.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:50:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:50.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:50:50.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:46 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:46' 2026-02-22T21:50:50.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:50.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:50.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:50.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:50.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:50.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:46' 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:50:46' 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:46' 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:50.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:50:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:50:46' 2026-02-22T21:50:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:50.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:50.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:46' 2026-02-22T21:50:50.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:50.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:50.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:50.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:50.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:50:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:50.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:50.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:50.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:50.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:50.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:50:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:51.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:51.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:50:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:51 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:50:51.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:46' 2026-02-22T21:50:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:50:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:50:51.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:50:51.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:50:51.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:50:51.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:51.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:50:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:50:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:50:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:50:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:51' 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:50:51' 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:51' 2026-02-22T21:50:51.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:50:51.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:50:51.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:50:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:50:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:50:46' 2026-02-22T21:50:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:50:51.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:50:51.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:51' 2026-02-22T21:50:51.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:50:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:50:51.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:50:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:50:51.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:50:51.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:50:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:50:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:50:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:51.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:51.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:50:52.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:52.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:52.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:52.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:52.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:52.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:52.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:52.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:52.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:53.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:53.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:50:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:53.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:50:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:50:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:55.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:55.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:55.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:55.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:55.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:55.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:55.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:55.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:55.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:50:55.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:50:55.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:50:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:55.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:50:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:50:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:59.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:59.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:50:59.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:50:59.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:50:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:50:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:50:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:50:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:59.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:50:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:50:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:50:59.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:59.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:50:59.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:50:59.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:59.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:59.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:50:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:50:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:50:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:50:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:50:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:50:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:50:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:50:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:50:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:50:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:50:59.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:50:59.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:50:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:50:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:50:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:50:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:50:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:59.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:50:59.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:50:59.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:50:59.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:50:59.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:50:59.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:59.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:50:59.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:50:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:50:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:50:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:50:59.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:50:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:50:59.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:59.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:50:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:59.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:50:59.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=34, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:59.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=34, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:59.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:59.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=34, entry_tid=0' 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:50:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:50:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:50:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:50:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:50:59.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:50:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:50:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:50:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=34, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=34, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:50:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:50:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=34, entry_tid=0' 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=34, entry_tid=0' -a 'object_number=0, tag_tid=34, entry_tid=0' = 'object_number=0, tag_tid=34, entry_tid=0' 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:50:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:51:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:00.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:00.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:56 2026-02-22T21:51:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":31},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":34}} 2026-02-22T21:51:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:56' 2026-02-22T21:51:00.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:51:00.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:51:00.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:00.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:51:00.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:00.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:00.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:00.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:00.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:00.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:00.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:00.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:00.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:56' 2026-02-22T21:51:00.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:50:56' 2026-02-22T21:51:00.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:00.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:00.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:00.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:00.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:56' 2026-02-22T21:51:00.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:00.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:00.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:50:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":31},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":34}}2026-02-22 21:50:56' 2026-02-22T21:51:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:00.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:56' 2026-02-22T21:51:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:51:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:51:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:51:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:51:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:01.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:01.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":31},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":34}} 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:56 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:50:56' 2026-02-22T21:51:01.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:51:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:51:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:01.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":31},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":34}} 2026-02-22T21:51:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:01.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:01.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:01.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:01.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:01.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:56' 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:50:56' 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:50:56' 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:01.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:01.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:01.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":31},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":34}}45054508mirror.0trial1042026-02-22 21:50:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:50:56' 2026-02-22T21:51:01.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:01.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:01.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:50:56' 2026-02-22T21:51:01.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:51:01.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:01.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:51:01.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:01.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:01.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:01.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:51:01.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:01.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:51:01.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:51:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:51:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:01.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:02.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:02.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:02.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:02.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:51:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:02.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:51:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:04.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:04.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:04.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:04.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:04.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:04.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:04.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:04.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:04.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:05.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:05.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:05.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:05.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:05.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:51:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:06.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:01 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:51:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:03' 2026-02-22T21:51:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:06.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:06.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:06.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:06.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:06.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:06.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:06.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:06.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:06.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:06.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:01' 2026-02-22T21:51:06.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:51:01' 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:01' 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:06.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:51:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:51:03' 2026-02-22T21:51:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:06.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:01' 2026-02-22T21:51:06.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:06.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:06.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:51:06.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:06.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:51:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:07.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:06 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:06' 2026-02-22T21:51:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:07.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:07.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:07.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:06' 2026-02-22T21:51:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:51:06' 2026-02-22T21:51:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:07.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:06' 2026-02-22T21:51:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:07.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:51:06cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:51:06' 2026-02-22T21:51:07.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:07.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:06' 2026-02-22T21:51:07.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:51:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:51:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:51:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:51:07.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:51:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:08.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:51:08.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:08.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:08.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:08.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:08.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:08.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:51:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:10.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:10.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:10.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:10.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:10.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:10.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:10.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:51:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:51:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:51:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:14.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:14.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:14.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:14.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:14.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:14.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:14.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:14.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:14.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:51:14.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:51:14.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:14.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:51:14.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:14.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:51:14.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:14.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:14.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:14.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:14.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:14.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:14.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:51:14.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:51:14.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:14.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:14.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:14.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:14.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:51:14.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:14.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=38, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:14.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=38, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:14.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:14.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=38, entry_tid=0' 2026-02-22T21:51:14.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:14.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:51:14.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:14.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=38, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=38, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:14.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:14.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=38, entry_tid=0' 2026-02-22T21:51:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=38, entry_tid=0' -a 'object_number=0, tag_tid=38, entry_tid=0' = 'object_number=0, tag_tid=38, entry_tid=0' 2026-02-22T21:51:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:51:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:14.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:14.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:15.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:15.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:11 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":35},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":38}} 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:11' 2026-02-22T21:51:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:51:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:51:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:51:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:15.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:15.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:11' 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:51:11' 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:11' 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:15.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:51:11cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":35},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":38}}2026-02-22 21:51:11' 2026-02-22T21:51:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:15.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:15.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:11' 2026-02-22T21:51:15.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:15.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:51:15.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:15.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:51:15.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:16.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:16.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":35},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":38}} 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:11 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:16' 2026-02-22T21:51:16.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:51:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:51:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":35},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":38}} 2026-02-22T21:51:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:16.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:11' 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:51:11' 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:11' 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:16.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:16.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":35},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":38}}43184322mirror.0trial1042026-02-22 21:51:11cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:51:16' 2026-02-22T21:51:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:16.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:16.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:11' 2026-02-22T21:51:16.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:51:16.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:51:16.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:51:16.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:51:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:16.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:51:16.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:16.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:51:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:16.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:51:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:17.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:17.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:17.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:17.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:17.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:17.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:17.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:17.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:51:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:17.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:51:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:19.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:19.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:19.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:20.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:20.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:20.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:20.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:20.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:20.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:51:20.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:20.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:51:20.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:20.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:21.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:21.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:51:21.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:18 2026-02-22T21:51:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:16' 2026-02-22T21:51:21.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:51:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:21.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:23.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:23.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:21 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:21' 2026-02-22T21:51:23.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:23.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:23.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:23.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:23.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:23.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:23.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:23.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:23.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:23.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:23.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:23.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:23.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:21' 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:51:21' 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:21' 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:23.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:51:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:51:21' 2026-02-22T21:51:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:23.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:23.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:21' 2026-02-22T21:51:23.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:23.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:23.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:23.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:23.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:24.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:21 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:21' 2026-02-22T21:51:24.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:24.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:24.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:24.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:21' 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:51:21' 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:21' 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:51:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:51:21' 2026-02-22T21:51:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:24.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:21' 2026-02-22T21:51:24.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:24.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:51:24.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:51:24.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:51:24.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:24.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:24.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:51:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:25.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:25.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:25.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:25.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:25.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:25.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:25.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:25.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:25.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:25.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:25.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:51:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:25.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:51:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:27.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:27.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:27.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:27.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:27.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:27.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:27.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:27.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:27.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:27.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:51:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:51:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:51:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:27.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:51:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:51:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:51:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:27.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:51:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:51:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:51:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:51:27.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:51:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:51:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:27.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:51:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:51:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:51:27.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:27.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:51:27.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:27.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:27.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:27.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:27.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:27.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:27.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:51:27.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:51:27.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:27.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:51:27.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=39, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:27.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=39, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:27.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:27.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=39, entry_tid=0' 2026-02-22T21:51:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:51:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:27.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:27.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:51:27.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:27.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=39, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:27.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=39, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:27.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=39, entry_tid=0' 2026-02-22T21:51:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=39, entry_tid=0' -a 'object_number=0, tag_tid=39, entry_tid=0' = 'object_number=0, tag_tid=39, entry_tid=0' 2026-02-22T21:51:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:27.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:27.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:51:28.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:28.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:28.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:26 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":36},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":39}} 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:26' 2026-02-22T21:51:28.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:51:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:51:28.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:51:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:28.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:28.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:28.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:26' 2026-02-22T21:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:51:26' 2026-02-22T21:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:28.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:26' 2026-02-22T21:51:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:28.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:51:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":36},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":39}}2026-02-22 21:51:26' 2026-02-22T21:51:28.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:28.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:26' 2026-02-22T21:51:28.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:28.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:51:28.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:28.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:51:28.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:28.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:29.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:29.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:29.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":36},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":39}} 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:26 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:26' 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:51:29.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":36},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":39}} 2026-02-22T21:51:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:29.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:29.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:26' 2026-02-22T21:51:29.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:51:26' 2026-02-22T21:51:29.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:29.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:29.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:29.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:29.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:26' 2026-02-22T21:51:29.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:29.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:29.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":36},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":39}}45054508mirror.0trial1042026-02-22 21:51:26cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:51:26' 2026-02-22T21:51:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:29.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:26' 2026-02-22T21:51:29.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:29.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:51:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:29.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:29.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:51:29.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:29.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:51:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:29.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:30.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:30.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:30.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:30.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:30.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:30.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:30.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:30.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:30.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:31.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:31.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:51:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:31.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:33.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:51:33.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:33.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:33.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:33.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:33.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:33.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:33.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:33.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:33.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:33.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:33.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:33.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:33.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:33.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:33.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:33.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:33.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:51:33.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:33.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:34.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:34.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:34.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:31 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:51:34.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:31' 2026-02-22T21:51:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:34.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:34.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:34.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:34.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:34.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:34.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:31' 2026-02-22T21:51:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:51:31' 2026-02-22T21:51:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:34.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:31' 2026-02-22T21:51:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:34.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:34.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:51:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:51:31' 2026-02-22T21:51:34.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:34.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:31' 2026-02-22T21:51:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:34.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:34.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:34.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:34.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:35.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:35.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:51:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:31 2026-02-22T21:51:35.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:31' 2026-02-22T21:51:35.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:51:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:35.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:37.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:37.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:36 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:37.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:36' 2026-02-22T21:51:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:37.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:37.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:37.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:37.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:36' 2026-02-22T21:51:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:51:36' 2026-02-22T21:51:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:37.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:36' 2026-02-22T21:51:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:37.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:37.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:51:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:51:36' 2026-02-22T21:51:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:37.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:37.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:36' 2026-02-22T21:51:37.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:37.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:37.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:51:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:37.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:51:37.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:51:37.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:51:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:51:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:51:37.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:51:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:37.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:38.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:38.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:38.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:38.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:38.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:38.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:38.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:51:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:38.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:51:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:40.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:40.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:40.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:40.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:40.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:40.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:40.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:40.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:40.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:40.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:40.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:40.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:51:40.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:40.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:51:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:51:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:44.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:44.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:44.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:44.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:44.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:44.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:44.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:44.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:44.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:44.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:44.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:44.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:51:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:51:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:51:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:51:44.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:51:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:51:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:44.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:51:44.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:51:44.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:51:44.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:44.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:51:44.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:51:44.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:44.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:44.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:51:44.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:44.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=43, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:44.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=43, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:44.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:44.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=43, entry_tid=0' 2026-02-22T21:51:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:51:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:44.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:51:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:51:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:44.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:51:44.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=43, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=43, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:51:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=43, entry_tid=0' 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=43, entry_tid=0' -a 'object_number=0, tag_tid=43, entry_tid=0' = 'object_number=0, tag_tid=43, entry_tid=0' 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:45.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:41 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":40},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43}} 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:41' 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:51:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:45.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:51:45.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:45.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:41' 2026-02-22T21:51:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:51:41' 2026-02-22T21:51:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:45.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:41' 2026-02-22T21:51:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:45.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:51:41cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":40},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43}}2026-02-22 21:51:41' 2026-02-22T21:51:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:45.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:41' 2026-02-22T21:51:45.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:45.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:51:45.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:45.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:51:45.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:51:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:45.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:45.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:51:46.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:46.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":8.0,"entries_behind_primary":0,"entries_per_second":0.33,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43}} 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:46 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:46.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:46' 2026-02-22T21:51:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:51:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:51:46.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":8.0,"entries_behind_primary":0,"entries_per_second":0.33,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43}} 2026-02-22T21:51:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:46.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:46.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:46.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:46' 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:51:46' 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:46' 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:46.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":8.0,"entries_behind_primary":0,"entries_per_second":0.33,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":43}}43184322mirror.0trial1042026-02-22 21:51:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:51:46' 2026-02-22T21:51:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:46.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:46.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:46' 2026-02-22T21:51:46.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:51:46.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:51:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:51:46.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:51:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:51:46.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:51:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:46.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:51:47.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:47.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:51:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:47.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:51:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:48.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:48.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:48.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:48.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:48.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:48.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:48.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:48.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:48.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:48.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:48.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:48.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:51:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:48.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:51:50.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:51:50.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:51:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:51:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:50.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:51:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:50.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:51:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:50.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:50.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:50.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:50.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:51:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:51:50.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:50.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:50.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:50.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:51.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:48 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:46' 2026-02-22T21:51:51.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:51:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:51.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:53.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:53.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:51 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:51' 2026-02-22T21:51:53.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:53.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:53.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:53.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:53.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:53.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:51' 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:51:51' 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:51' 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:53.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:51:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:51:51' 2026-02-22T21:51:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:53.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:51' 2026-02-22T21:51:53.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:53.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:51:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:54.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:54.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:51:54.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:51 2026-02-22T21:51:54.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:51' 2026-02-22T21:51:54.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:51:54.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:51:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:54.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:51:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:54.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:54.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:54.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:54.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:51' 2026-02-22T21:51:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:51:51' 2026-02-22T21:51:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:51:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:51' 2026-02-22T21:51:54.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:54.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:51:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:51:51' 2026-02-22T21:51:54.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:54.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:54.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:51' 2026-02-22T21:51:54.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:51:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:51:54.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:51:54.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:51:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:51:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:54.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:51:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:51:54.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:54.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:51:55.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:55.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:55.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:55.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:55.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:55.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:55.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:55.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:55.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:55.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:55.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:55.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:51:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:51:55.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:55.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:51:55.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:55.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:51:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:51:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:51:57.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:57.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:51:57.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:51:57.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:51:57.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:51:57.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:51:57.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:51:57.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:57.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:57.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:57.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:57.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:51:57.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:51:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:51:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:51:57.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:51:57.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:51:57.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:51:57.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:51:57.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:51:57.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:51:57.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:57.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:57.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:51:57.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:51:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:51:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:51:57.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:51:57.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:51:57.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:51:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:51:57.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:51:57.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:57.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:51:57.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=44, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:57.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=44, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:58.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=44, entry_tid=0' 2026-02-22T21:51:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:51:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:58.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:51:58.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:51:58.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:58.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:51:58.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:51:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:51:58.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:51:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=44, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=44, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:51:58.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:51:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=44, entry_tid=0' 2026-02-22T21:51:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=44, entry_tid=0' -a 'object_number=0, tag_tid=44, entry_tid=0' = 'object_number=0, tag_tid=44, entry_tid=0' 2026-02-22T21:51:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:51:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:58.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:58.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:51:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:51:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:59.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:59.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:51:59.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:51:59.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:51:59.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:51:59.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:56 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":41},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":44}} 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:56' 2026-02-22T21:51:59.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:51:59.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:51:59.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:51:59.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:51:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:51:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:59.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:51:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:51:59.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:51:59.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:51:59.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:51:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:56' 2026-02-22T21:51:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:51:56' 2026-02-22T21:51:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:51:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:59.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:51:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:56' 2026-02-22T21:51:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:51:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:51:59.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:51:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:51:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":41},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":44}}2026-02-22 21:51:56' 2026-02-22T21:51:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:51:59.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:51:59.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:56' 2026-02-22T21:51:59.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:51:59.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:51:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:51:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:51:59.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:51:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:51:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:51:59.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:51:59.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:00.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:00.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:52:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":41},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":44}} 2026-02-22T21:52:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:00.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:56 2026-02-22T21:52:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:52:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:51:56' 2026-02-22T21:52:00.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:52:00.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:52:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:00.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":41},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":44}} 2026-02-22T21:52:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:00.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:56' 2026-02-22T21:52:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:51:56' 2026-02-22T21:52:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:51:56' 2026-02-22T21:52:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:00.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":41},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":44}}45054508mirror.0trial1042026-02-22 21:51:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:51:56' 2026-02-22T21:52:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:00.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:51:56' 2026-02-22T21:52:00.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:00.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:52:00.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:00.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:00.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:52:00.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:52:00.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:00.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:00.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:52:01.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:01.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:01.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:01.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:01.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:01.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:01.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:01.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:01.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:52:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:52:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:01.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:52:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:52:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:52:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:01.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:52:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:03.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:03.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:03.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:03.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:03.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:03.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:03.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:03.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:03.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:03.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:03.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:52:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:03.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:52:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:03.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:03.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:03.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:04.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:52:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:04.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:04.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:04.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:01 2026-02-22T21:52:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:52:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:01' 2026-02-22T21:52:04.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:04.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:04.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:04.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:04.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:01' 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:52:01' 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:01' 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:04.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:52:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:52:01' 2026-02-22T21:52:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:04.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:04.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:01' 2026-02-22T21:52:04.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:05.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:05.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:01 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:01' 2026-02-22T21:52:05.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:52:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:05.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:07.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:07.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:07.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:06 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:07.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:06' 2026-02-22T21:52:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:07.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:07.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:07.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:07.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:06' 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:52:06' 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:06' 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:07.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:07.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:07.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:52:06cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:52:06' 2026-02-22T21:52:07.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:07.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:07.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:06' 2026-02-22T21:52:07.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:07.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:07.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:07.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:52:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:52:07.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:52:07.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:52:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:52:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:52:07.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:52:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:08.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:08.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:08.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:08.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:08.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:08.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:08.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:08.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:08.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:08.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:08.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:08.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:52:08.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:08.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:10.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:52:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:10.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:10.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:10.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:10.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:10.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:52:10.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:10.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:52:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:52:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:14.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:14.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:14.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:14.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:14.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:14.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:14.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:14.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:52:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:52:14.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:14.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:52:14.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:52:14.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:52:14.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:52:15.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:52:15.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:52:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:52:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:52:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:15.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:52:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:52:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:52:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:52:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:52:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:15.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:15.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:15.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:15.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:52:15.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:52:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:15.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:52:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:52:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:52:15.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:52:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:52:15.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:52:15.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:52:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:52:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:52:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=48, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:15.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=48, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:15.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:15.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=48, entry_tid=0' 2026-02-22T21:52:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:52:15.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:52:15.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:52:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:52:15.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:52:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:52:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:52:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=48, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:15.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=48, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:15.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=48, entry_tid=0' 2026-02-22T21:52:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=48, entry_tid=0' -a 'object_number=0, tag_tid=48, entry_tid=0' = 'object_number=0, tag_tid=48, entry_tid=0' 2026-02-22T21:52:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:52:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:15.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:15.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:52:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:16.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:16.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:11 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":45},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48}} 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:11' 2026-02-22T21:52:16.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:52:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:52:16.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:52:16.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:52:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:52:16.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:16.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:16.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:16.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:16.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:11' 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:52:11' 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:11' 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:16.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:16.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:52:11cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":45},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48}}2026-02-22 21:52:11' 2026-02-22T21:52:16.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:16.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:16.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:11' 2026-02-22T21:52:16.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:16.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:52:16.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:52:16.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:52:16.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:16.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:17.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48}} 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:16 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:17.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:52:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:16' 2026-02-22T21:52:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:52:17.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:52:17.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:17.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48}} 2026-02-22T21:52:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:17.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:17.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:16' 2026-02-22T21:52:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:52:16' 2026-02-22T21:52:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:17.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:16' 2026-02-22T21:52:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:17.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:17.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":48}}43184322mirror.0trial1042026-02-22 21:52:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:52:16' 2026-02-22T21:52:17.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:17.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:16' 2026-02-22T21:52:17.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:17.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:52:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:52:17.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:17.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:52:17.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:52:17.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:17.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:52:18.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:18.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:18.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:18.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:18.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:18.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:18.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:18.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:18.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:18.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:18.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:18.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:18.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:52:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:52:18.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:52:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:52:18.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:52:18.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:18.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:52:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:20.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:20.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:20.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:20.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:20.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:20.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:20.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:20.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:20.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:20.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:20.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:20.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:20.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:20.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:20.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:20.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:20.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:20.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:52:20.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:20.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:52:20.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:20.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:21.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:21.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:21.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:19 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:21' 2026-02-22T21:52:21.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:52:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:21.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:23.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:23.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:23.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:21 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:21' 2026-02-22T21:52:23.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:23.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:23.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:23.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:23.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:23.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:21' 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:52:21' 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:21' 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:23.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:23.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:52:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:52:21' 2026-02-22T21:52:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:23.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:21' 2026-02-22T21:52:23.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:23.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:23.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:23.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:52:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:23.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:23.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:24.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:24.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:24.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:21 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:21' 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:24.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:24.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:24.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:24.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:24.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:21' 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:52:21' 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:21' 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:24.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:52:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:52:21' 2026-02-22T21:52:25.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:25.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:21' 2026-02-22T21:52:25.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:25.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:25.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:25.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:25.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:52:25.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:52:25.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:52:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:25.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:26.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:26.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:26.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:26.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:26.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:26.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:52:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:28.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:52:28.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:28.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:28.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:28.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:28.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:28.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:28.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:28.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:28.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:28.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:52:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:52:28.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:28.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:52:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:52:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:52:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:52:28.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:52:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:52:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:52:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:52:28.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:52:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:52:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:52:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:52:28.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:52:28.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:52:28.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:52:28.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:52:28.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:28.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:52:28.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:28.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:52:28.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:52:28.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:52:28.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:52:28.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:28.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:52:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:52:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:52:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:52:28.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:52:28.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:28.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:52:28.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:52:28.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:52:28.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=49, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:52:28.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=49, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:52:28.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:28.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=49, entry_tid=0' 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:52:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:52:28.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:52:28.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:52:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:52:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:52:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=49, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:52:28.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=49, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:52:28.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=49, entry_tid=0' 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=49, entry_tid=0' -a 'object_number=0, tag_tid=49, entry_tid=0' = 'object_number=0, tag_tid=49, entry_tid=0' 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:28.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:29.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:52:29.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:29.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:26 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":46},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":49}} 2026-02-22T21:52:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:26' 2026-02-22T21:52:29.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:52:29.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:52:29.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:52:29.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:52:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:52:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:26' 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:52:26' 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:26' 2026-02-22T21:52:29.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:29.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:29.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:52:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":46},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":49}}2026-02-22 21:52:26' 2026-02-22T21:52:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:29.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:29.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:26' 2026-02-22T21:52:29.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:29.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:52:29.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:52:29.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:52:29.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:29.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:30.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":46},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":49}} 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:26 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:26' 2026-02-22T21:52:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:52:30.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:52:30.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:30.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":46},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":49}} 2026-02-22T21:52:30.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:30.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:30.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:30.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:30.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:30.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:30.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:26' 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:52:26' 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:26' 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:30.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":46},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":49}}45054508mirror.0trial1042026-02-22 21:52:26cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:52:26' 2026-02-22T21:52:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:30.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:26' 2026-02-22T21:52:30.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:52:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:30.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:52:30.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:52:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:30.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:52:30.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:52:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:52:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:30.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:52:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:31.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:31.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:31.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:31.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:31.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:31.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:31.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:31.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:52:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:52:31.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:31.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:52:31.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:52:31.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:52:31.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:31.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:33.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:52:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:33.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:33.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:33.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:33.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:33.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:33.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:33.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:33.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:33.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:34.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:34.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:52:34.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:34.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:34.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:35.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:35.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:31 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:35.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:32' 2026-02-22T21:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:35.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:35.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:35.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:35.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:31' 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:52:31' 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:31' 2026-02-22T21:52:35.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:35.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:35.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:52:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:52:32' 2026-02-22T21:52:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:31' 2026-02-22T21:52:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:35.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:35.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:35.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:52:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:36.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:36.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:32 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:31' 2026-02-22T21:52:36.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:52:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:36.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:38.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:38.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:38.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:38.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:38.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:38.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:38.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:38.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:38.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:36 2026-02-22T21:52:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:36' 2026-02-22T21:52:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:38.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:38.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:38.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:38.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:38.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:36' 2026-02-22T21:52:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:52:36' 2026-02-22T21:52:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:38.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:36' 2026-02-22T21:52:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:38.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:38.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:52:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:52:36' 2026-02-22T21:52:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:38.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:36' 2026-02-22T21:52:38.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:38.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:38.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:38.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:38.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:38.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:52:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:52:38.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:52:38.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:52:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:52:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:52:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:38.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:52:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:52:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:39.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:39.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:39.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:39.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:39.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:39.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:39.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:39.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:39.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:39.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:52:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:39.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:41.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:41.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:41.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:41.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:41.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:41.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:41.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:41.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:41.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:52:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:41.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:52:45.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:52:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:45.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:45.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:45.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:45.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:45.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:45.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:45.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:45.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:52:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:52:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:45.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:52:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:52:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:52:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:52:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:52:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:52:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:52:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:52:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:52:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:52:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:52:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:52:45.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:45.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:45.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:45.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:45.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:52:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:52:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:45.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:52:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:52:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:52:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:52:45.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:52:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:52:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:45.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:52:45.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:52:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:52:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=53, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:45.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=53, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:45.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:45.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=53, entry_tid=0' 2026-02-22T21:52:45.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:52:45.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:45.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:45.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:52:45.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:52:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:52:45.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:45.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:52:45.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:52:45.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:52:45.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=53, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:45.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=53, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:52:45.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:52:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=53, entry_tid=0' 2026-02-22T21:52:45.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=53, entry_tid=0' -a 'object_number=0, tag_tid=53, entry_tid=0' = 'object_number=0, tag_tid=53, entry_tid=0' 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:45.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:46.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:46 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":50},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53}} 2026-02-22T21:52:46.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:41' 2026-02-22T21:52:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:52:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:52:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:52:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:52:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:46.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:52:46.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:46.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:46.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:46.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:46' 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:52:46' 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:46' 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:46.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:52:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":50},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53}}2026-02-22 21:52:41' 2026-02-22T21:52:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:46.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:46' 2026-02-22T21:52:46.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:46.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:52:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:52:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:52:46.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:52:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:52:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:46.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:46.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:47.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:47.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:47.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:47.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53}} 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:46 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:52:47.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:46' 2026-02-22T21:52:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:52:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:52:47.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53}} 2026-02-22T21:52:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:47.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:47.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:47.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:46' 2026-02-22T21:52:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:52:46' 2026-02-22T21:52:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:47.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:52:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:46' 2026-02-22T21:52:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:47.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":53}}43184322mirror.0trial1042026-02-22 21:52:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:52:46' 2026-02-22T21:52:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:47.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:46' 2026-02-22T21:52:47.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:52:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:52:47.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:52:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:52:47.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:52:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:52:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:47.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:52:47.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:52:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:52:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:47.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:47.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:48.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:48.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:48.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:48.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:49.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:52:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:52:49.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:52:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:52:49.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:52:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:52:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:52:51.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:51.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:52:51.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:51.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:51.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:51.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:52:51.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:52:51.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:51.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:51.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:51.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:51.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:52.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:51 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:52.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:52:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:51' 2026-02-22T21:52:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:52.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:52.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:52.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:52.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:52.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:51' 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:52:51' 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:51' 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:52.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:52.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:52:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:52:51' 2026-02-22T21:52:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:52.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:51' 2026-02-22T21:52:52.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:52.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:52.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:52.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:52.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:52:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:53.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:52:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:52:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:53.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:52:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:52:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:51 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:51' 2026-02-22T21:52:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:52:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:52:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:52:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:52:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:52:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:52:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:52:53.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:52:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:52:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:52:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:51' 2026-02-22T21:52:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:52:51' 2026-02-22T21:52:53.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:52:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:51' 2026-02-22T21:52:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:52:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:52:53.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:52:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:52:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:52:51' 2026-02-22T21:52:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:52:53.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:52:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:51' 2026-02-22T21:52:53.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:52:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:52:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:52:53.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:52:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:52:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:52:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:52:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:52:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:52:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:54.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:54.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:54.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:54.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:54.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:52:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:54.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:52:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:52:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:52:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:52:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:52:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:52:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:52:56.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:52:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:52:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:52:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:52:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:52:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:52:56.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:52:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:52:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:52:56.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:52:56.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:52:56.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:52:56.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:52:56.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:52:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:56.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:52:56.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:52:56.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:52:56.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:52:56.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:52:56.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:52:56.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:52:56.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:52:56.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:53:00.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:53:00.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:00.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:00.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:00.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:00.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:00.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:00.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:00.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:00.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:00.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:00.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:00.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:53:00.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:53:00.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:53:00.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:53:00.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:00.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:53:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:53:00.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:53:00.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:00.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:53:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:53:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:00.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:00.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:00.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:00.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:00.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:53:00.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=54, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:00.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=54, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:00.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=54, entry_tid=0' 2026-02-22T21:53:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:53:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:00.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:53:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:53:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:00.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:53:00.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=54, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:00.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=54, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:00.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:00.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=54, entry_tid=0' 2026-02-22T21:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=54, entry_tid=0' -a 'object_number=0, tag_tid=54, entry_tid=0' = 'object_number=0, tag_tid=54, entry_tid=0' 2026-02-22T21:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:00.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:00.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:01.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:01.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:01.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:56 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":51},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":54}} 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:56' 2026-02-22T21:53:01.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:53:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:53:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:01.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:53:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:01.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:01.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:01.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:01.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:56' 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:52:56' 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:56' 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:01.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:01.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:52:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":51},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":54}}2026-02-22 21:52:56' 2026-02-22T21:53:01.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:01.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:56' 2026-02-22T21:53:01.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:53:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:01.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:53:01.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:01.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:01.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:53:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:02.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:02.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":51},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":54}} 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:52:56 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:01' 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:53:02.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:53:02.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:02.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":51},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":54}} 2026-02-22T21:53:02.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:02.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:02.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:02.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:02.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:02.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:56' 2026-02-22T21:53:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:52:56' 2026-02-22T21:53:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:02.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:52:56' 2026-02-22T21:53:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:02.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":51},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":54}}45054508mirror.0trial1042026-02-22 21:52:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:53:01' 2026-02-22T21:53:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:02.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:02.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:52:56' 2026-02-22T21:53:02.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:02.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:53:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:53:02.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:02.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:53:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:02.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:53:03.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:53:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:53:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:03.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:03.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:53:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:04.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:04.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:04.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:04.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:04.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:04.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:04.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:04.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:04.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:53:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:04.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:53:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:06.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:06.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:06.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:06.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:06.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:53:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:53:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:07.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:07.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:06 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:06' 2026-02-22T21:53:07.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:07.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:07.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:07.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:07.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:07.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:06' 2026-02-22T21:53:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:53:06' 2026-02-22T21:53:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:07.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:06' 2026-02-22T21:53:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:07.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:53:06cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:53:06' 2026-02-22T21:53:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:06' 2026-02-22T21:53:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:07.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:07.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:08.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:06 2026-02-22T21:53:08.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:06' 2026-02-22T21:53:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:08.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:08.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:08.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:08.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:08.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:06' 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:53:06' 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:06' 2026-02-22T21:53:08.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:08.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:53:06cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:53:06' 2026-02-22T21:53:08.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:06' 2026-02-22T21:53:08.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:08.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:08.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:08.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:53:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:53:08.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:53:08.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:53:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:53:08.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:53:09.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:09.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:09.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:09.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:09.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:09.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:09.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:09.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:09.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:09.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:09.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:53:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:09.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:53:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:11.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:11.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:11.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:11.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:11.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:11.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:11.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:11.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:11.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:11.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:11.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:11.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:11.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:11.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:53:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:11.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:53:15.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:53:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:15.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:15.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:15.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:15.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:15.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:15.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:15.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:15.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:15.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:15.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:15.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:15.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:15.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:15.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:53:15.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:53:15.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:53:15.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:15.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:53:15.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:53:15.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:53:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:15.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:53:15.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:15.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:53:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:15.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:53:15.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:15.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:15.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:15.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:15.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=58, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:15.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=58, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:15.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=58, entry_tid=0' 2026-02-22T21:53:15.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:53:15.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=58, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:15.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=58, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:15.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=58, entry_tid=0' 2026-02-22T21:53:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=58, entry_tid=0' -a 'object_number=0, tag_tid=58, entry_tid=0' = 'object_number=0, tag_tid=58, entry_tid=0' 2026-02-22T21:53:15.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:15.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:16.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:16.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:16.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:16 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":55},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58}} 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:11' 2026-02-22T21:53:16.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:53:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:53:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:53:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:16.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:16.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:16' 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:53:16' 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:16' 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:16.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:53:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":55},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58}}2026-02-22 21:53:11' 2026-02-22T21:53:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:16.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:16.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:16' 2026-02-22T21:53:16.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:16.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:53:16.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:16.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:53:16.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:16.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:16.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:53:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:17.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:17.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58}} 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:16 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:17.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:16' 2026-02-22T21:53:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:53:17.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:53:17.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:17.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58}} 2026-02-22T21:53:17.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:17.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:17.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:16' 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:53:16' 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:16' 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:17.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:17.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:17.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":58}}43184322mirror.0trial1042026-02-22 21:53:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:53:16' 2026-02-22T21:53:17.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:17.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:17.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:16' 2026-02-22T21:53:17.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:53:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:17.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:53:17.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:53:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:53:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:17.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:17.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:53:18.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:18.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:53:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:18.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:53:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:19.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:19.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:19.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:19.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:19.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:19.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:19.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:19.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:19.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:19.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:19.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:19.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:19.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:19.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:53:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:19.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:53:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:21.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:21.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:21.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:21.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:21.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:21.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:21.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:21.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:21.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:21.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:21.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:21.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:53:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:21.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:21.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:53:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:22.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:22.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:21 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:22.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:21' 2026-02-22T21:53:22.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:22.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:22.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:22.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:22.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:22.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:22.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:22.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:22.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:22.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:21' 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:53:21' 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:21' 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:53:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:53:21' 2026-02-22T21:53:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:22.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:22.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:21' 2026-02-22T21:53:22.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:22.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:22.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:22.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:22.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:22.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:53:23.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:23.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:23.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:21 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:21' 2026-02-22T21:53:23.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:23.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:23.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:23.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:23.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:23.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:23.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:23.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:23.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:21' 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:53:21' 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:21' 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:23.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:53:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:53:21' 2026-02-22T21:53:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:23.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:21' 2026-02-22T21:53:23.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:23.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:23.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:53:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:53:23.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:53:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:24.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:24.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:24.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:24.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:24.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:24.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:24.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:24.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:24.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:24.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:24.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:53:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:24.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:53:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:26.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:26.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:26.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:26.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:26.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:26.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:26.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:26.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:26.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:26.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:26.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:53:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:26.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:53:30.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:53:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:30.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:30.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:30.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:30.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:30.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:53:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:53:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:53:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:53:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:53:30.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:30.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:53:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:53:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:53:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:53:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:53:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:53:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:53:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:53:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:53:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:53:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:30.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:30.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:30.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:53:30.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=59, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:30.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=59, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:30.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:30.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=59, entry_tid=0' 2026-02-22T21:53:30.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:30.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:30.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:53:30.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:30.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:30.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:30.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:30.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=59, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:30.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=59, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:30.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=59, entry_tid=0' 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=59, entry_tid=0' -a 'object_number=0, tag_tid=59, entry_tid=0' = 'object_number=0, tag_tid=59, entry_tid=0' 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:30.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:30.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:53:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:31.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:26 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":56},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":59}} 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:26' 2026-02-22T21:53:31.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:53:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:53:31.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:53:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:31.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:31.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:31.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:26' 2026-02-22T21:53:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:53:26' 2026-02-22T21:53:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:31.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:26' 2026-02-22T21:53:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:31.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:53:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":56},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":59}}2026-02-22 21:53:26' 2026-02-22T21:53:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:31.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:31' 2026-02-22T21:53:31.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:53:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:31.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:53:31.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:53:31.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:31.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:53:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:32.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:32.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:32.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":56},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":59}} 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:26 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:32.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:31' 2026-02-22T21:53:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:53:32.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:53:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:32.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":56},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":59}} 2026-02-22T21:53:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:32.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:32.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:32.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:26' 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:53:26' 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:26' 2026-02-22T21:53:32.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:32.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":56},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":59}}45054508mirror.0trial1042026-02-22 21:53:26cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:53:31' 2026-02-22T21:53:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:33.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:33.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:26' 2026-02-22T21:53:33.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:33.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:53:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:53:33.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:33.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:33.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:33.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:53:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:53:33.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:53:33.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:33.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:53:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:34.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:34.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:34.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:34.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:34.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:34.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:34.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:34.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:34.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:34.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:34.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:34.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:53:34.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:34.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:53:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:36.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:36.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:36.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:36.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:36.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:36.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:36.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:36.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:36.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:36.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:53:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:53:36.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:36.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:37.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:36 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:36' 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:37.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:37.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:37.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:37.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:37.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:37.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:37.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:36' 2026-02-22T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:53:36' 2026-02-22T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:37.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:36' 2026-02-22T21:53:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:37.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:53:36cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:53:36' 2026-02-22T21:53:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:37.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:37.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:36' 2026-02-22T21:53:37.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:37.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:37.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:37.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:37.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:53:38.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:38.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:38.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:38.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:38.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:36 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:36' 2026-02-22T21:53:38.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:38.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:38.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:38.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:38.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:38.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:38.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:38.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:36' 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:53:36' 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:36' 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:38.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:38.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:53:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:53:36' 2026-02-22T21:53:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:38.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:38.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:36' 2026-02-22T21:53:38.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:38.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:38.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:38.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:38.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:53:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:38.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:53:38.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:53:38.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:53:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:53:38.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:53:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:39.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:39.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:39.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:39.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:39.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:39.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:39.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:39.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:39.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:39.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:39.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:39.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:39.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:39.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:53:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:39.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:53:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:53:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:53:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:53:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:45.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:45.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:45.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:45.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:45.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:45.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:45.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:45.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:45.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:53:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:53:45.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:53:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:53:45.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:53:45.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:53:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:53:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:53:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:53:45.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:53:45.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:45.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:53:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:53:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:45.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:45.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:45.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:53:45.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:53:45.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:53:45.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=63, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:45.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=63, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:45.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:45.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=63, entry_tid=0' 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:53:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:45.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:53:45.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=63, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:45.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=63, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:53:45.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=63, entry_tid=0' 2026-02-22T21:53:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=63, entry_tid=0' -a 'object_number=0, tag_tid=63, entry_tid=0' = 'object_number=0, tag_tid=63, entry_tid=0' 2026-02-22T21:53:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:53:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:45.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:45.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:53:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:46.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:46.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:46.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:46 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63}} 2026-02-22T21:53:46.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:46' 2026-02-22T21:53:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:53:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:53:46.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:53:46.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:46.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:46.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:46.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:46.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:46.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:46' 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:53:46' 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:46' 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:46.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:53:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63}}2026-02-22 21:53:46' 2026-02-22T21:53:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:46.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:46' 2026-02-22T21:53:46.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:46.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:53:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:53:46.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:46.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:46.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:47.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:47.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:48.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63}} 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:46 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:46' 2026-02-22T21:53:48.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:53:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:53:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:48.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63}} 2026-02-22T21:53:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:48.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:48.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:48.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:46' 2026-02-22T21:53:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:53:46' 2026-02-22T21:53:48.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:46' 2026-02-22T21:53:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:48.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:48.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":6.0,"entries_behind_primary":0,"entries_per_second":0.25,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":63}}43184322mirror.0trial1042026-02-22 21:53:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:53:46' 2026-02-22T21:53:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:48.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:46' 2026-02-22T21:53:48.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:48.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:53:48.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:48.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:53:48.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:53:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:53:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:53:48.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:53:48.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:48.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:48.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:53:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:49.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:49.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:49.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:49.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:49.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:49.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:49.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:49.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:49.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:49.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:49.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:49.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:49.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:53:49.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:49.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:51.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:53:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:53:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:51.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:51.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:53:51.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:53:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:53:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:53:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:53:51.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:51.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:53:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:51.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:53:51.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:51.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:51.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:51.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:51.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:53:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:53:51.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:51.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:52.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:51 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:51' 2026-02-22T21:53:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:52.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:51' 2026-02-22T21:53:52.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:53:51' 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:51' 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:52.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:53:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:53:51' 2026-02-22T21:53:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:51' 2026-02-22T21:53:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:52.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:52.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:52.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:52.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:53:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:53.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:53.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:53.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:53.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:51 2026-02-22T21:53:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:53:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:53:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:51' 2026-02-22T21:53:53.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:53:53.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:53:53.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:53.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:53.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:53.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:53.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:51' 2026-02-22T21:53:53.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:53:51' 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:51' 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:53.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:53:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:53:51' 2026-02-22T21:53:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:53.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:53.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:51' 2026-02-22T21:53:53.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:53.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:53:53.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:53:53.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:53:53.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:53:53.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:53:53.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:53:53.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:53.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:53.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:53:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:54.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:54.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:54.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:54.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:54.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:54.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:54.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:54.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:54.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:54.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:54.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:54.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:54.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:53:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:53:54.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:54.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:53:54.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:54.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:53:56.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:53:56.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:56.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:56.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:56.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:53:56.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:53:56.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:53:56.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:53:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:56.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:53:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:56.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:56.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:53:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:53:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:53:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:53:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:53:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:53:56.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:56.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:56.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:56.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:56.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:53:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:53:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:53:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:53:56.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:53:56.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:53:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:53:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:53:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:53:56.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:53:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:53:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:53:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:53:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:53:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:53:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:53:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:53:56.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:53:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:53:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:56.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:53:56.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:53:56.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:53:56.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:53:56.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:53:56.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:53:56.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:53:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:53:56.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:57.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:53:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:57.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:53:57.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:57.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=64, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:57.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=64, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:57.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:57.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=64, entry_tid=0' 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:53:57.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:53:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:53:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:53:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:53:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:53:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:57.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:53:57.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:57.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:53:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:53:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:53:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=64, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=64, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:53:57.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:53:57.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=64, entry_tid=0' 2026-02-22T21:53:57.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=64, entry_tid=0' -a 'object_number=0, tag_tid=64, entry_tid=0' = 'object_number=0, tag_tid=64, entry_tid=0' 2026-02-22T21:53:57.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:57.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:53:57.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:57.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:57.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:56 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":61},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":64}} 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:56' 2026-02-22T21:53:58.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:53:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:53:58.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:53:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:58.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:58.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:58.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:58.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:58.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:58.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:56' 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:53:56' 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:56' 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:58.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:58.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:53:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":61},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":64}}2026-02-22 21:53:56' 2026-02-22T21:53:58.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:58.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:56' 2026-02-22T21:53:58.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:58.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:53:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:53:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:53:58.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:53:58.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:58.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:53:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:53:59.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:53:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:53:59.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:59.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:53:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:53:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:53:59.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":61},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":64}} 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:56 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:53:56' 2026-02-22T21:53:59.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:53:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:53:59.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:53:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":61},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":64}} 2026-02-22T21:53:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:53:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:59.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:53:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:53:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:53:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:53:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:53:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:56' 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:53:56' 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:53:56' 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:53:59.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:53:59.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:53:59.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":61},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":64}}45054508mirror.0trial1042026-02-22 21:53:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:53:56' 2026-02-22T21:53:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:53:59.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:53:59.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:53:56' 2026-02-22T21:53:59.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:53:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:53:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:53:59.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:53:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:53:59.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:53:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:53:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:53:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:53:59.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:53:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:53:59.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:53:59.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:00.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:00.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:00.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:00.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:00.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:00.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:00.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:00.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:00.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:00.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:00.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:00.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:00.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:00.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:54:00.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:00.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:02.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:02.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:02.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:02.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:02.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:02.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:02.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:02.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:02.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:02.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:02.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:02.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:54:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:02.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:02.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:03.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:01 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:01' 2026-02-22T21:54:03.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:01' 2026-02-22T21:54:03.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:54:01' 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:01' 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:03.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:54:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:54:01' 2026-02-22T21:54:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:03.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:01' 2026-02-22T21:54:03.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:03.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:03.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:03.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:03.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:54:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:04.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:04.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:04.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:04.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:04.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:04.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:01 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:01' 2026-02-22T21:54:04.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:04.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:04.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:04.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:04.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:04.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:04.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:04.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:01' 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:54:01' 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:01' 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:04.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:54:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:54:01' 2026-02-22T21:54:04.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:04.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:01' 2026-02-22T21:54:04.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:04.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:04.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:04.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:54:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:54:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:54:04.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:54:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:54:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:05.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:05.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:05.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:05.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:05.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:05.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:05.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:05.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:05.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:05.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:05.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:05.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:54:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:54:07.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:07.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:07.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:07.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:07.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:07.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:07.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:07.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:07.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:07.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:54:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:07.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:54:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:54:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:54:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:54:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:07.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:54:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:54:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:54:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:07.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:54:07.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:07.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:07.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:54:08.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:08.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:08.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:54:08.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=68, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:08.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=68, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:08.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:08.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=68, entry_tid=0' 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:08.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:54:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:08.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:54:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:08.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=68, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:08.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=68, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:08.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=68, entry_tid=0' 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=68, entry_tid=0' -a 'object_number=0, tag_tid=68, entry_tid=0' = 'object_number=0, tag_tid=68, entry_tid=0' 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:08.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:08.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:54:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:09.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:09.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:09.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:09.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:06 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":65},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":68}} 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:06' 2026-02-22T21:54:09.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:54:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:54:09.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:54:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:09.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:09.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:09.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:06' 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:54:06' 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:06' 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:09.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:09.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:54:06cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":65},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":68}}2026-02-22 21:54:06' 2026-02-22T21:54:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:06' 2026-02-22T21:54:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:09.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:54:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:09.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:54:09.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:09.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:09.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:10.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:10.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":65},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":68}} 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:06 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:10.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:06' 2026-02-22T21:54:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:54:10.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:54:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:10.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":65},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":68}} 2026-02-22T21:54:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:10.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:10.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:06' 2026-02-22T21:54:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:54:06' 2026-02-22T21:54:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:10.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:06' 2026-02-22T21:54:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:10.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":65},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":68}}43184322mirror.0trial1042026-02-22 21:54:06cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:54:06' 2026-02-22T21:54:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:10.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:10.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:06' 2026-02-22T21:54:10.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:10.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:54:10.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:10.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:54:10.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:10.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:54:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:54:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:10.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:54:10.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:54:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:54:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:10.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:10.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:54:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:11.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:11.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:11.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:11.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:11.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:11.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:11.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:11.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:11.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:54:11.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:11.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:54:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:13.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:13.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:13.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:13.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:13.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:13.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:13.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:13.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:13.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:13.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:13.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:13.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:14.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:14.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:14.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:12 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:11' 2026-02-22T21:54:14.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:54:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:14.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:16.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:16.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:16.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:54:16.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:12 2026-02-22T21:54:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:11' 2026-02-22T21:54:16.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:54:16.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:16.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:20.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:20.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:20.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:16 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:16' 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:20.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:20.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:20.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:20.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:20.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:20.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:20.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:20.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:16' 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:54:16' 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:16' 2026-02-22T21:54:20.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:20.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:54:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:54:16' 2026-02-22T21:54:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:20.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:16' 2026-02-22T21:54:20.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:20.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:20.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:20.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:20.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:21.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:54:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:21.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:21.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:16 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:16' 2026-02-22T21:54:21.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:21.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:21.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:21.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:21.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:21.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:21.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:21.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:16' 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:54:16' 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:16' 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:21.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:54:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:54:16' 2026-02-22T21:54:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:21.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:21.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:16' 2026-02-22T21:54:21.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:21.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:21.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:21.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:21.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:21.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:21.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:21.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:54:21.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:54:21.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:54:21.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:54:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:54:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:21.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:21.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:54:22.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:22.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:22.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:22.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:22.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:22.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:22.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:22.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:22.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:22.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:22.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:54:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:22.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:22.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:54:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:54:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:54:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:54:22.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:54:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:54:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:54:22.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:54:22.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:22.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:54:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:54:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:54:22.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:22.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:22.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:54:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:54:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:22.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:22.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:54:22.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:54:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:54:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:54:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:22.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:22.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:54:22.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:23.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=69, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=69, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:23.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:23.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=69, entry_tid=0' 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:23.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:54:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:23.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:54:23.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:23.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=69, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=69, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:23.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=69, entry_tid=0' 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=69, entry_tid=0' -a 'object_number=0, tag_tid=69, entry_tid=0' = 'object_number=0, tag_tid=69, entry_tid=0' 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:23.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:23.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:24.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:24.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:24.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:24.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:21 2026-02-22T21:54:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":66},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":69}} 2026-02-22T21:54:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:21' 2026-02-22T21:54:24.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:54:24.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:54:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:24.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:54:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:24.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:24.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:24.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:21' 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:54:21' 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:21' 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:24.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:54:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":66},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":69}}2026-02-22 21:54:21' 2026-02-22T21:54:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:24.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:24.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:21' 2026-02-22T21:54:24.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:54:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:24.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:54:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:24.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:24.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:54:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:25.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:25.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":66},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":69}} 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:21 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:25.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:21' 2026-02-22T21:54:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:54:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:54:25.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:25.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":66},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":69}} 2026-02-22T21:54:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:25.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:25.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:25.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:25.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:25.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:21' 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:54:21' 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:21' 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:25.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":66},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":69}}45054508mirror.0trial1042026-02-22 21:54:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:54:21' 2026-02-22T21:54:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:25.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:21' 2026-02-22T21:54:25.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:54:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:25.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:54:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:54:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:54:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:54:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:54:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:25.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:54:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:25.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:26.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:26.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:26.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:26.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:26.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:26.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:26.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:26.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:26.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:26.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:26.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:54:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:28.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:28.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:28.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:28.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:28.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:28.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:28.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:28.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:28.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:54:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:54:28.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:28.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:29.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:29.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:29.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:26 2026-02-22T21:54:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:54:29.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:26' 2026-02-22T21:54:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:29.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:29.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:29.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:26' 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:54:26' 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:26' 2026-02-22T21:54:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:29.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:54:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:54:26' 2026-02-22T21:54:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:29.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:26' 2026-02-22T21:54:29.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:29.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:29.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:29.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:29.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:29.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:29.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:54:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:30.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:30.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:54:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:26 2026-02-22T21:54:30.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:26' 2026-02-22T21:54:30.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:54:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:54:32.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:32.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:32.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:31 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:32.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:31' 2026-02-22T21:54:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:32.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:32.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:32.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:32.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:32.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:32.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:32.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:31' 2026-02-22T21:54:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:54:31' 2026-02-22T21:54:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:32.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:31' 2026-02-22T21:54:32.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:32.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:32.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:32.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:54:31cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:54:31' 2026-02-22T21:54:32.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:32.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:32.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:31' 2026-02-22T21:54:32.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:54:32.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:54:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:54:32.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:32.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:54:33.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:33.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:33.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:33.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:33.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:33.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:33.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:33.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:33.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:33.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:33.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:54:33.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:33.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:54:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:35.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:35.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:35.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:35.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:35.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:35.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:35.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:35.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:35.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:35.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:35.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:54:35.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:35.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:54:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:54:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:39.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:39.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:39.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:39.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:39.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:39.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:54:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:54:39.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:54:39.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:54:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:54:39.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:54:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:54:39.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:54:39.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:54:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:54:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:39.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:54:39.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:39.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:39.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:40.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:54:40.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:40.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:54:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:40.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:54:40.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=73, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:40.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=73, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:40.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:40.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=73, entry_tid=0' 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:40.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:54:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:54:40.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:40.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=73, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:40.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=73, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:54:40.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=73, entry_tid=0' 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=73, entry_tid=0' -a 'object_number=0, tag_tid=73, entry_tid=0' = 'object_number=0, tag_tid=73, entry_tid=0' 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:40.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:40.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:40.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:54:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:41.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:41.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:41.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:36 2026-02-22T21:54:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":70},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":73}} 2026-02-22T21:54:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:36' 2026-02-22T21:54:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:54:41.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:54:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:41.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:54:41.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:41.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:41.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:41.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:41.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:41.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:41.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:41.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:36' 2026-02-22T21:54:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:54:36' 2026-02-22T21:54:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:41.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:36' 2026-02-22T21:54:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:41.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:54:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":70},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":73}}2026-02-22 21:54:36' 2026-02-22T21:54:41.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:41.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:36' 2026-02-22T21:54:41.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:41.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:54:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:41.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:54:41.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:41.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:41.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:42.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":70},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":73}} 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:36 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:41' 2026-02-22T21:54:42.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:54:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:54:42.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":70},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":73}} 2026-02-22T21:54:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:42.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:42.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:42.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:42.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:36' 2026-02-22T21:54:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:54:36' 2026-02-22T21:54:42.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:36' 2026-02-22T21:54:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:42.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":70},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":73}}43184322mirror.0trial1042026-02-22 21:54:36cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:54:41' 2026-02-22T21:54:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:42.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:42.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:36' 2026-02-22T21:54:42.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:42.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:54:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:54:42.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:54:42.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:54:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:54:42.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:54:42.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:42.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:43.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:54:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:43.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:43.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:43.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:43.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:43.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:43.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:43.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:43.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:43.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:43.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:43.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:54:43.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:43.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:54:45.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:54:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:54:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:45.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:45.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:54:45.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:54:45.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:45.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:54:45.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:45.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:45.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:54:45.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:45.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:54:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:54:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:46.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:46.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:46.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:46.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:44 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:46.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:41' 2026-02-22T21:54:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:54:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:48.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:48.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:48.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:48.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:48.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:48.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:48.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:48.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:48.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:48.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:46 2026-02-22T21:54:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:46' 2026-02-22T21:54:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:48.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:48.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:48.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:48.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:48.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:48.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:48.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:48.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:48.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:48.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:48.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:48.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:46' 2026-02-22T21:54:48.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:54:46' 2026-02-22T21:54:48.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:48.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:48.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:46' 2026-02-22T21:54:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:48.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:54:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:54:46' 2026-02-22T21:54:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:48.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:46' 2026-02-22T21:54:48.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:48.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:48.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:48.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:54:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:48.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:48.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:48.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:48.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:48.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:48.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:49.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:54:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:49.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:49.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:46 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:46' 2026-02-22T21:54:49.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:49.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:49.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:49.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:49.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:49.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:49.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:49.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:49.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:49.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:46' 2026-02-22T21:54:49.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:54:46' 2026-02-22T21:54:49.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:49.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:49.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:49.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:49.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:46' 2026-02-22T21:54:49.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:49.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:49.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:54:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:54:46' 2026-02-22T21:54:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:49.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:49.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:46' 2026-02-22T21:54:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:54:49.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:54:49.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:54:49.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:49.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:50.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:54:50.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:50.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:50.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:50.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:50.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:50.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:50.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:50.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:50.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:50.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:50.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:50.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:50.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:50.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:50.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:54:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:50.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:54:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:52.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:52.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:54:52.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:52.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:52.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:52.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:52.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:52.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:52.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:52.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:52.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:52.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:52.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:52.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:52.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:54:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:54:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:54:52.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:54:52.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:54:52.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:54:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:54:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:54:53.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:54:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:54:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:54:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:54:53.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:54:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:53.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:53.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:53.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:54:53.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:53.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:54:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:53.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:53.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:53.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:54:53.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:53.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=74, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:53.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=74, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:53.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=74, entry_tid=0' 2026-02-22T21:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:53.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:54:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:54:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:54:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:54:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:54:53.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:54:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:54:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:54:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:54:53.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:53.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:54:53.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:54:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:54:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=74, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=74, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:54:53.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:54:53.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=74, entry_tid=0' 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=74, entry_tid=0' -a 'object_number=0, tag_tid=74, entry_tid=0' = 'object_number=0, tag_tid=74, entry_tid=0' 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:53.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:54:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:54.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:54.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:54.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:54.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:54.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:51 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":71},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":74}} 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:51' 2026-02-22T21:54:54.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:54:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:54:54.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:54:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:54.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:54.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:54.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:51' 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:54:51' 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:51' 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:54.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:54.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:54:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":71},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":74}}2026-02-22 21:54:51' 2026-02-22T21:54:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:51' 2026-02-22T21:54:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:54:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:54:54.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:54:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:55.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:55.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:54:55.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":71},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":74}} 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:51 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:55.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:54:55.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:51' 2026-02-22T21:54:55.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:54:55.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:54:55.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:55.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":71},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":74}} 2026-02-22T21:54:55.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:55.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:55.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:51' 2026-02-22T21:54:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:54:51' 2026-02-22T21:54:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:54:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:51' 2026-02-22T21:54:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:55.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":71},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":74}}45054508mirror.0trial1042026-02-22 21:54:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:54:51' 2026-02-22T21:54:55.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:55.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:55.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:51' 2026-02-22T21:54:55.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:55.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:54:55.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:54:55.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:54:55.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:54:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:54:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:54:55.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:54:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:54:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:55.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:55.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:54:55.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:54:55.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:55.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:56.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:56.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:56.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:56.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:56.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:54:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:54:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:54:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:54:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:54:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:54:58.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:54:58.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:54:58.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:54:58.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:54:58.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:54:58.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:54:58.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:58.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:58.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:54:58.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:54:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:54:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:54:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:54:58.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:54:58.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:58.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:54:59.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:54:59.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:59.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:59.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:54:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:54:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:54:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:54:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:54:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:54:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:56 2026-02-22T21:54:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:54:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:54:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:54:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:54:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:56' 2026-02-22T21:54:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:54:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:54:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:54:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:54:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:54:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:54:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:54:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:54:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:54:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:54:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:56' 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:54:56' 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:54:56' 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:54:59.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:54:59.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:54:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:54:56' 2026-02-22T21:54:59.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:54:59.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:54:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:54:56' 2026-02-22T21:54:59.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:54:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:54:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:54:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:54:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:54:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:54:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:54:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:54:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:54:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:54:59.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:54:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:54:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:54:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:54:59.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:55:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:00.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:00.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:57 2026-02-22T21:55:00.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:54:56' 2026-02-22T21:55:00.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:55:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:00.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:55:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:02.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:02.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:01 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:01' 2026-02-22T21:55:02.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:02.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:55:02.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:55:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:02.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:02.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:02.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:01' 2026-02-22T21:55:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:55:01' 2026-02-22T21:55:02.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:01' 2026-02-22T21:55:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:02.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:02.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:55:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:55:01' 2026-02-22T21:55:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:02.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:01' 2026-02-22T21:55:02.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:55:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:02.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:55:02.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:02.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:55:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:55:02.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:55:02.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:55:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:55:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:02.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:02.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:55:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:03.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:03.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:03.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:03.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:03.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:03.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:03.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:03.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:03.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:03.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:03.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:03.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:03.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:05.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:55:05.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:05.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:05.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:05.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:05.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:05.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:05.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:05.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:05.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:05.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:05.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:05.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:05.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:06.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:06.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:06.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:06.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:06.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:55:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:55:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:10.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:10.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:10.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:10.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:10.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:10.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:10.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:10.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:55:10.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:55:10.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:10.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:55:10.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:10.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:55:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:55:10.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:10.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=78, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:10.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=78, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:10.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:10.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=78, entry_tid=0' 2026-02-22T21:55:10.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:10.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:10.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:55:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=78, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:10.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=78, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=78, entry_tid=0' 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=78, entry_tid=0' -a 'object_number=0, tag_tid=78, entry_tid=0' = 'object_number=0, tag_tid=78, entry_tid=0' 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:10.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:10.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:10.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:10.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:10.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:11.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:55:11.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:11.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:11.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:11.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:11.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:06 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":75},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":78}} 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:06' 2026-02-22T21:55:11.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:55:11.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:55:11.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:55:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:11.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:11.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:11.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:11.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:11.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:11.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:06' 2026-02-22T21:55:11.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:55:06' 2026-02-22T21:55:11.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:06' 2026-02-22T21:55:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:11.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:55:11cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":75},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":78}}2026-02-22 21:55:06' 2026-02-22T21:55:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:11.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:11.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:11' 2026-02-22T21:55:11.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:55:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:11.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:55:11.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:11.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:12.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:12.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:55:12.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":75},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":78}} 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:06 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:12.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:11' 2026-02-22T21:55:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:55:12.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:55:12.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:12.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":75},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":78}} 2026-02-22T21:55:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:12.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:12.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:12.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:12.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:06' 2026-02-22T21:55:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:55:06' 2026-02-22T21:55:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:12.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:06' 2026-02-22T21:55:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:12.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:12.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":75},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":78}}43184322mirror.0trial1042026-02-22 21:55:06cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:55:11' 2026-02-22T21:55:12.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:12.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:12.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:06' 2026-02-22T21:55:12.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:55:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:12.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:55:12.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:12.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:55:13.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:55:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:55:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:13.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:55:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:55:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:13.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:55:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:14.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:14.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:14.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:14.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:14.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:14.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:14.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:14.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:14.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:14.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:14.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:14.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:14.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:14.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:55:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:14.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:55:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:55:16.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:16.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:16.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:16.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:16.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:16.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:16.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:16.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:16.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:55:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:17.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:17.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:55:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:17.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:16 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:16' 2026-02-22T21:55:17.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:55:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:17.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:17.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:17.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:17.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:16' 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:55:16' 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:17.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:16' 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:17.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:55:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:55:16' 2026-02-22T21:55:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:16' 2026-02-22T21:55:17.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:17.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:55:17.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:17.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:55:17.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:17.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:17.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:55:18.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:18.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:16 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:18.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:16' 2026-02-22T21:55:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:18.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:55:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:18.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:55:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:18.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:18.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:18.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:16' 2026-02-22T21:55:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:55:16' 2026-02-22T21:55:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:18.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:16' 2026-02-22T21:55:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:18.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:55:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:55:16' 2026-02-22T21:55:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:18.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:18.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:16' 2026-02-22T21:55:18.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:18.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:55:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:55:18.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:55:18.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:55:18.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:55:18.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:55:18.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:55:18.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:18.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:18.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:18.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:55:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:55:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:18.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:55:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:19.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:19.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:19.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:19.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:19.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:19.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:19.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:19.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:19.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:55:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:21.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:21.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:21.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:21.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:21.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:21.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:21.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:21.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:21.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:21.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:21.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:21.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:55:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:55:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:25.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:25.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:25.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:25.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:25.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:25.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:25.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:25.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:25.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:25.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:55:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:55:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:55:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:55:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:55:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:25.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:55:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:55:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:55:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:25.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:55:25.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:25.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:25.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:25.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:25.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:55:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:55:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:25.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:55:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:55:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:25.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:25.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:25.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:55:25.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=79, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:25.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=79, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:25.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:25.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=79, entry_tid=0' 2026-02-22T21:55:25.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:25.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:55:25.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:25.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:25.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:25.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:25.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=79, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:25.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=79, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:25.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=79, entry_tid=0' 2026-02-22T21:55:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=79, entry_tid=0' -a 'object_number=0, tag_tid=79, entry_tid=0' = 'object_number=0, tag_tid=79, entry_tid=0' 2026-02-22T21:55:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:26.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:55:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:26.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:21 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":76},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":79}} 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:21' 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:55:26.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:55:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:26.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:26.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:26.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:26.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:21' 2026-02-22T21:55:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:55:21' 2026-02-22T21:55:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:26.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:21' 2026-02-22T21:55:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:26.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:55:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":76},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":79}}2026-02-22 21:55:21' 2026-02-22T21:55:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:26.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:26.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:21' 2026-02-22T21:55:26.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:26.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:26.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:26.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:55:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:27.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:27.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:27.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":76},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":79}} 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:21 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:27.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:26' 2026-02-22T21:55:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:55:27.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:55:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":76},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":79}} 2026-02-22T21:55:27.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:27.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:27.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:21' 2026-02-22T21:55:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:55:21' 2026-02-22T21:55:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:27.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:21' 2026-02-22T21:55:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:27.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:27.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":76},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":79}}45054508mirror.0trial1042026-02-22 21:55:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:55:26' 2026-02-22T21:55:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:28.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:21' 2026-02-22T21:55:28.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:55:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:28.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:28.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:55:28.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:55:28.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:28.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:55:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:29.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:29.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:29.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:29.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:29.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:29.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:29.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:29.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:29.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:29.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:29.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:55:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:29.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:55:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:31.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:55:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:31.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:31.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:31.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:31.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:31.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:31.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:31.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:31.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:31.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:31.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:31.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:32.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:32.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:55:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:31 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:31' 2026-02-22T21:55:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:55:32.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:55:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:32.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:32.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:32.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:32.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:32.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:32.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:31' 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:55:31' 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:31' 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:32.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:55:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:55:31' 2026-02-22T21:55:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:32.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:32.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:31' 2026-02-22T21:55:32.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:32.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:55:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:55:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:32.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:32.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:55:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:32.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:32.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:33.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:33.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:31 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:33.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:31' 2026-02-22T21:55:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:33.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:55:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:33.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:55:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:33.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:33.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:33.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:31' 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:55:31' 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:31' 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:55:31cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:55:31' 2026-02-22T21:55:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:31' 2026-02-22T21:55:33.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:55:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:33.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:55:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:33.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:55:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:55:33.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:55:33.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:33.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:34.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:34.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:34.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:34.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:34.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:34.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:34.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:34.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:34.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:34.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:34.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:34.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:55:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:36.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:36.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:36.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:36.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:36.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:36.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:36.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:55:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:55:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:40.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:40.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:40.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:40.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:40.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:40.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:40.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:40.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:40.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:40.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:40.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:40.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:55:40.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:55:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:55:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:40.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:40.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:55:40.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:55:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:55:40.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:40.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:40.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:55:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:55:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:40.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:55:40.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:55:40.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:40.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:55:40.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:40.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:40.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:40.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:40.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=83, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:40.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=83, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:40.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:40.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=83, entry_tid=0' 2026-02-22T21:55:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:55:40.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:40.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:40.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:40.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:55:40.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:40.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=83, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=83, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:55:40.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=83, entry_tid=0' 2026-02-22T21:55:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=83, entry_tid=0' -a 'object_number=0, tag_tid=83, entry_tid=0' = 'object_number=0, tag_tid=83, entry_tid=0' 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:40.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:41.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:41 2026-02-22T21:55:41.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":80},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":83}} 2026-02-22T21:55:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:36' 2026-02-22T21:55:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:55:41.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:55:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:41.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:55:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:41.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:41.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:41' 2026-02-22T21:55:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:55:41' 2026-02-22T21:55:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:41.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:41' 2026-02-22T21:55:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:41.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:41.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:55:41cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":80},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":83}}2026-02-22 21:55:36' 2026-02-22T21:55:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:41.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:41' 2026-02-22T21:55:41.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:41.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:55:41.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:41.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:55:41.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:55:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:41.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:41.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:41.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:41.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:41.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:42.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:42.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:55:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:43.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":80},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":83}} 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:36 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:41' 2026-02-22T21:55:43.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:55:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:55:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:43.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":80},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":83}} 2026-02-22T21:55:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:43.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:43.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:43.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:36' 2026-02-22T21:55:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:55:36' 2026-02-22T21:55:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:43.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:36' 2026-02-22T21:55:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:43.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":80},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":83}}43184322mirror.0trial1042026-02-22 21:55:36cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:55:41' 2026-02-22T21:55:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:43.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:43.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:36' 2026-02-22T21:55:43.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:55:43.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:55:43.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:55:43.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:55:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:43.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:55:43.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:55:43.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:43.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:44.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:44.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:44.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:44.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:44.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:44.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:44.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:44.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:44.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:44.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:44.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:55:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:44.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:46.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:55:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:55:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:46.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:46.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:55:46.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:55:46.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:55:46.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:46.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:55:46.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:55:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:46.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:46.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:46.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:55:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:55:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:46.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:46.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:55:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:47.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:47.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:55:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:47.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:46 2026-02-22T21:55:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:55:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:47.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:46' 2026-02-22T21:55:47.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:47.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:55:47.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:47.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:55:47.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:47.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:47.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:47.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:46' 2026-02-22T21:55:47.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:55:46' 2026-02-22T21:55:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:47.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:46' 2026-02-22T21:55:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:47.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:55:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:55:46' 2026-02-22T21:55:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:47.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:47.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:46' 2026-02-22T21:55:47.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:47.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:55:47.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:47.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:55:47.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:47.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:48.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:48.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:48.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:46 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:48.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:55:48.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:55:48.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:46' 2026-02-22T21:55:48.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:55:48.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:55:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:48.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:55:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:48.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:48.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:46' 2026-02-22T21:55:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:55:46' 2026-02-22T21:55:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:55:48.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:46' 2026-02-22T21:55:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:48.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:55:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:55:46' 2026-02-22T21:55:48.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:48.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:48.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:46' 2026-02-22T21:55:48.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:48.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:55:48.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:55:48.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:55:48.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:55:48.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:55:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:55:48.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:55:48.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:55:48.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:48.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:55:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:49.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:49.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:49.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:49.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:49.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:49.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:49.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:49.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:49.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:49.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:49.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:49.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:49.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:49.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:49.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:55:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:55:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:51.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:51.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:51.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:51.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:51.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:51.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:51.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:51.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:51.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:51.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:51.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:51.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:51.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:51.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:55:51.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:55:51.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:51.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:55:51.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:51.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:55:55.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:55:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:55:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:55.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:55.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:55.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:55.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:55.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:55.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:55.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:55.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:55.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:55.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:55.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:55.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:55:55.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:55.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:55:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:55:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:55:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:55:55.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:55:55.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:55:55.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:55.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:55:55.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:55:55.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:55:55.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:55.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:55.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:55.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:55:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:55.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:55:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=84, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:55.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=84, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:55.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:55.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=84, entry_tid=0' 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:55:55.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:55:55.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:55.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:55:55.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:55:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:55:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:55:55.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=84, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=84, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:55:55.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:55:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=84, entry_tid=0' 2026-02-22T21:55:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=84, entry_tid=0' -a 'object_number=0, tag_tid=84, entry_tid=0' = 'object_number=0, tag_tid=84, entry_tid=0' 2026-02-22T21:55:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:55:55.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:55.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:55:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:56.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:56.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:55:56.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:56 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":81},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":84}} 2026-02-22T21:55:56.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:51' 2026-02-22T21:55:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:55:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:55:56.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:56.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:56.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:56.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:56.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:56' 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:55:56' 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:56' 2026-02-22T21:55:56.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:56.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:56.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:57.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:55:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":81},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":84}}2026-02-22 21:55:51' 2026-02-22T21:55:57.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:57.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:57.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:56' 2026-02-22T21:55:57.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:55:57.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:55:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:55:57.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:55:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:55:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:57.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:57.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:58.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:58.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":81},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":84}} 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:51 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:55:56' 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:55:58.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:55:58.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:55:58.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":81},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":84}} 2026-02-22T21:55:58.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:55:58.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:58.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:55:58.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:55:58.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:55:58.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:55:58.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:55:58.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:51' 2026-02-22T21:55:58.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:55:51' 2026-02-22T21:55:58.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:58.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:58.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:58.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:55:58.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:55:51' 2026-02-22T21:55:58.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:55:58.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:55:58.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:55:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":81},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":84}}45054508mirror.0trial1042026-02-22 21:55:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:55:56' 2026-02-22T21:55:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:55:58.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:55:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:55:51' 2026-02-22T21:55:58.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:55:58.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:55:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:55:58.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:55:58.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:58.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:55:58.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:55:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:55:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:58.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:58.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:55:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:55:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:55:59.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:55:59.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:55:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:55:59.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:55:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:55:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:55:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:55:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:55:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:55:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:55:59.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:55:59.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:55:59.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:55:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:55:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:55:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:55:59.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:55:59.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:55:59.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:55:59.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:55:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:55:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:55:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:55:59.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:55:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:55:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:55:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:55:59.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:56:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:01.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:01.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:01.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:01.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:01.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:01.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:01.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:01.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:01.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:01.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:01.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:56:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:56:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:01.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:56:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:02.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:02.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:02.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:01 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:01' 2026-02-22T21:56:02.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:02.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:02.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:02.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:02.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:02.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:02.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:02.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:01' 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:56:01' 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:01' 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:02.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:56:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:56:01' 2026-02-22T21:56:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:02.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:01' 2026-02-22T21:56:02.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:02.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:03.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:03.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:03.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:01 2026-02-22T21:56:03.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:01' 2026-02-22T21:56:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:03.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:03.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:03.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:03.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:03.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:03.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:03.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:03.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:01' 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:56:01' 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:01' 2026-02-22T21:56:03.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:03.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:03.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:56:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:56:01' 2026-02-22T21:56:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:03.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:01' 2026-02-22T21:56:03.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:03.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:03.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:03.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:03.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:56:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:56:03.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:56:03.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:56:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:56:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:56:03.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:56:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:03.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:56:04.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:04.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:04.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:04.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:04.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:04.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:04.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:04.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:04.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:04.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:04.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:04.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:04.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:04.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:56:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:04.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:56:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:06.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:06.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:06.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:06.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:06.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:06.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:06.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:06.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:06.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:06.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:06.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:06.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:06.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:56:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:06.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:56:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:56:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:10.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:10.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:10.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:10.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:10.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:10.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:56:10.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:56:10.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:10.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:56:10.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:56:10.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:56:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:56:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:56:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:56:10.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:56:10.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:56:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:56:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:10.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:56:10.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:10.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:10.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:56:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:10.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:10.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:10.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:56:10.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=88, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:10.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=88, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:10.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=88, entry_tid=0' 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:10.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:56:10.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=88, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:10.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=88, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:10.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=88, entry_tid=0' 2026-02-22T21:56:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=88, entry_tid=0' -a 'object_number=0, tag_tid=88, entry_tid=0' = 'object_number=0, tag_tid=88, entry_tid=0' 2026-02-22T21:56:10.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:56:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:11.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:11.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:12.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:11 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":85},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":88}} 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:06' 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:56:12.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:12.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:56:12.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:12.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:12.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:12.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:12.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:11' 2026-02-22T21:56:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:56:11' 2026-02-22T21:56:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:12.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:11' 2026-02-22T21:56:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:12.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:12.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:56:11cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":85},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":88}}2026-02-22 21:56:06' 2026-02-22T21:56:12.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:12.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:12.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:11' 2026-02-22T21:56:12.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:12.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:56:12.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:12.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:56:12.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:12.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:13.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":85},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":88}} 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:06 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:13.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:11' 2026-02-22T21:56:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:56:13.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:56:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:13.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":85},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":88}} 2026-02-22T21:56:13.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:13.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:13.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:13.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:13.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:13.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:13.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:06' 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:56:06' 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:06' 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:13.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":85},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":88}}43184322mirror.0trial1042026-02-22 21:56:06cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:56:11' 2026-02-22T21:56:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:13.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:06' 2026-02-22T21:56:13.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:13.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:56:13.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:13.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:56:13.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:13.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:56:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:56:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:13.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:56:13.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:56:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:56:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:13.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:56:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:56:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:13.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:14.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:56:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:14.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:14.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:14.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:14.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:14.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:14.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:14.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:14.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:14.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:14.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:14.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:14.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:14.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:14.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:14.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:56:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:56:14.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:14.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:56:15.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:15.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:15.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:14 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:11' 2026-02-22T21:56:15.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:56:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:15.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:17.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:16 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:16' 2026-02-22T21:56:17.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:17.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:17.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:17.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:17.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:17.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:16' 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:56:16' 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:16' 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:17.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:17.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:56:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:56:16' 2026-02-22T21:56:17.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:17.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:17.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:16' 2026-02-22T21:56:17.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:17.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:17.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:17.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:17.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:17.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:56:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:18.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:18.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:18.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:18.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:18.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:18.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:18.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:18.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:16 2026-02-22T21:56:18.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:18.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:18.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:18.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:18.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:16' 2026-02-22T21:56:18.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:18.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:18.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:18.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:18.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:18.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:16' 2026-02-22T21:56:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:56:16' 2026-02-22T21:56:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:18.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:16' 2026-02-22T21:56:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:18.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:18.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:56:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:56:16' 2026-02-22T21:56:18.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:18.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:16' 2026-02-22T21:56:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:18.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:18.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:56:18.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:56:18.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:18.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:56:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:18.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:56:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:19.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:19.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:19.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:19.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:19.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:19.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:19.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:19.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:19.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:19.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:19.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:19.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:19.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:56:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:19.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:21.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:21.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:21.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:21.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:21.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:21.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:21.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:21.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:21.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:21.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:21.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:56:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:21.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:56:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:56:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:25.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:25.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:25.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:25.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:25.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:25.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:25.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:25.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:56:25.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:56:25.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:25.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:56:25.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:56:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:56:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:56:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:56:25.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:56:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:56:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:56:25.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:56:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:56:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:56:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:56:25.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:56:25.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:56:25.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:25.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:25.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:56:25.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:56:25.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:25.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:25.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:25.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:25.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:56:25.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:25.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:25.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:56:25.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=89, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:25.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=89, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:25.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:25.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=89, entry_tid=0' 2026-02-22T21:56:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:56:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:56:25.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:56:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:25.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:56:25.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=89, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:26.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=89, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:26.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:26.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=89, entry_tid=0' 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=89, entry_tid=0' -a 'object_number=0, tag_tid=89, entry_tid=0' = 'object_number=0, tag_tid=89, entry_tid=0' 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:26.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:26.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:27.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:26 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":86},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":89}} 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:21' 2026-02-22T21:56:27.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:56:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:56:27.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:27.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:27.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:27.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:26' 2026-02-22T21:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:56:26' 2026-02-22T21:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:27.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:26' 2026-02-22T21:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:27.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:56:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":86},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":89}}2026-02-22 21:56:21' 2026-02-22T21:56:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:27.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:27.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:26' 2026-02-22T21:56:27.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:56:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:27.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:56:27.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:27.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:28.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:28.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:28.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":86},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":89}} 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:21 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:28.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:26' 2026-02-22T21:56:28.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:56:28.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:56:28.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:28.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":86},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":89}} 2026-02-22T21:56:28.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:28.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:28.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:28.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:28.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:21' 2026-02-22T21:56:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:56:21' 2026-02-22T21:56:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:28.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:21' 2026-02-22T21:56:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:28.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:28.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":86},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":89}}45054508mirror.0trial1042026-02-22 21:56:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:56:26' 2026-02-22T21:56:28.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:28.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:21' 2026-02-22T21:56:28.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:56:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:28.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:56:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:28.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:56:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:28.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:56:28.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:28.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:56:29.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:29.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:29.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:29.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:29.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:29.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:29.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:29.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:29.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:29.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:29.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:29.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:29.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:56:29.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:56:29.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:56:29.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:56:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:56:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:29.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:56:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:31.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:31.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:31.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:31.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:31.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:31.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:31.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:31.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:31.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:31.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:31.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:31.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:56:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:32.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:31 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:32.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:31' 2026-02-22T21:56:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:32.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:32.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:32.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:32.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:32.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:32.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:31' 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:56:31' 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:31' 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:32.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:32.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:32.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:56:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:56:31' 2026-02-22T21:56:32.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:32.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:32.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:31' 2026-02-22T21:56:32.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:32.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:32.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:32.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:32.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:33.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:33.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:31 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:31' 2026-02-22T21:56:33.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:33.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:33.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:33.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:33.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:33.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:33.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:33.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:31' 2026-02-22T21:56:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:56:31' 2026-02-22T21:56:33.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:31' 2026-02-22T21:56:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:33.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:56:31cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:56:31' 2026-02-22T21:56:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:33.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:31' 2026-02-22T21:56:33.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:33.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:33.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:56:33.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:56:33.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:33.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:33.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:34.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:34.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:34.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:34.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:34.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:34.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:34.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:34.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:34.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:34.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:34.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:34.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:34.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:34.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:56:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:34.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:36.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:56:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:36.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:36.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:36.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:36.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:36.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:36.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:36.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:36.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:36.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:36.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:36.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:36.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:36.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:56:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:36.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:56:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:56:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:40.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:40.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:40.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:40.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:40.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:40.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:40.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:56:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:56:40.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:40.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:56:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:56:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:56:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:56:40.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:56:40.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:56:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:56:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:56:40.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:56:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:56:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:41.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:41.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:56:41.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:41.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:56:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:56:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:41.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:41.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:56:41.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:56:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:41.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:41.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:56:41.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=93, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=93, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=93, entry_tid=0' 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:41.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:56:41.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:41.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:41.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:41.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:41.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:56:41.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:41.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=93, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:41.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=93, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:56:41.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=93, entry_tid=0' 2026-02-22T21:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=93, entry_tid=0' -a 'object_number=0, tag_tid=93, entry_tid=0' = 'object_number=0, tag_tid=93, entry_tid=0' 2026-02-22T21:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:41.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:41.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:42.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:42.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:42.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:41 2026-02-22T21:56:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":90},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":93}} 2026-02-22T21:56:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:36' 2026-02-22T21:56:42.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:56:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:56:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:42.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:56:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:42.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:42.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:41' 2026-02-22T21:56:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:56:41' 2026-02-22T21:56:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:42.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:41' 2026-02-22T21:56:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:42.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:56:41cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":90},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":93}}2026-02-22 21:56:36' 2026-02-22T21:56:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:42.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:42.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:41' 2026-02-22T21:56:42.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:42.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:56:42.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:42.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:56:42.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:56:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:42.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:42.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:43.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:43.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":90},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":93}} 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:36 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:43.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:41' 2026-02-22T21:56:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:56:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:56:43.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":90},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":93}} 2026-02-22T21:56:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:43.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:43.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:43.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:43.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:43.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:43.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:36' 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:56:36' 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:36' 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:43.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":90},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":93}}43184322mirror.0trial1042026-02-22 21:56:36cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:56:41' 2026-02-22T21:56:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:43.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:36' 2026-02-22T21:56:43.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:56:43.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:56:43.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:43.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:56:43.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:43.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:56:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:44.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:44.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:44.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:44.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:44.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:44.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:44.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:44.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:44.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:44.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:56:44.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:56:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:44.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:56:44.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:56:44.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:56:44.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:44.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:56:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:46.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:56:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:56:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:46.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:56:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:56:46.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:46.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:46.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:56:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:56:46.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:46.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:46.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:46.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:46.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:46.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:56:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:47.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:47.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:47.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:47.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:46 2026-02-22T21:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:46' 2026-02-22T21:56:47.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:47.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:47.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:47.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:47.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:47.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:46' 2026-02-22T21:56:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:56:46' 2026-02-22T21:56:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:46' 2026-02-22T21:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:47.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:47.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:47.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:56:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:56:46' 2026-02-22T21:56:47.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:47.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:47.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:46' 2026-02-22T21:56:47.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:47.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:47.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:47.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:47.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:56:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:47.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:47.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:48.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:56:48.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:48.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:48.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:48.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:48.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:46 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:48.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:46' 2026-02-22T21:56:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:48.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:48.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:48.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:46' 2026-02-22T21:56:48.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:56:46' 2026-02-22T21:56:48.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:48.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:48.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:48.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:46' 2026-02-22T21:56:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:48.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:48.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:56:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:56:46' 2026-02-22T21:56:48.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:48.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:46' 2026-02-22T21:56:48.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:48.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:48.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:56:48.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:56:48.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:56:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:56:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:48.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:56:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:48.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:56:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:48.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:49.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:49.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:49.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:49.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:49.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:49.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:49.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:49.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:49.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:49.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:49.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:49.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:49.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:49.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:56:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:49.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:56:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:56:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:51.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:51.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:51.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:51.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:51.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:51.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:56:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:56:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:51.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:56:51.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:56:51.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:56:51.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:56:51.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:56:52.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:52.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:52.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:56:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:56:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:52.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:52.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:52.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:56:52.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:56:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:56:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:52.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:56:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:52.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:56:52.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=94, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=94, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=94, entry_tid=0' 2026-02-22T21:56:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:56:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:56:52.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:56:52.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:52.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:56:52.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:56:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:56:52.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:56:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=94, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=94, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:56:52.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:56:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=94, entry_tid=0' 2026-02-22T21:56:52.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=94, entry_tid=0' -a 'object_number=0, tag_tid=94, entry_tid=0' = 'object_number=0, tag_tid=94, entry_tid=0' 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:52.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:53.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:53.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:53.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:51 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":91},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":94}} 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:51' 2026-02-22T21:56:53.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:56:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:56:53.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:56:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:53.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:53.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:53.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:51' 2026-02-22T21:56:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:56:51' 2026-02-22T21:56:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:53.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:56:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:51' 2026-02-22T21:56:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:53.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:56:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":91},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":94}}2026-02-22 21:56:51' 2026-02-22T21:56:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:53.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:51' 2026-02-22T21:56:53.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:56:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:56:53.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:56:53.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:56:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:56:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:53.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:53.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:54.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:56:54.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:54.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:56:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":91},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":94}} 2026-02-22T21:56:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:51 2026-02-22T21:56:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:56:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:56:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:51' 2026-02-22T21:56:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:56:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:56:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":91},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":94}} 2026-02-22T21:56:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:54.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:54.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:54.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:51' 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:56:51' 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:51' 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:54.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:54.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":91},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":94}}45054508mirror.0trial1042026-02-22 21:56:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:56:51' 2026-02-22T21:56:54.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:54.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:51' 2026-02-22T21:56:54.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:54.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:56:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:56:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:56:54.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:56:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:56:54.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:56:54.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:56:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:56:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:56:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:54.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:56:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:56:55.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:55.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:55.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:55.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:55.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:55.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:55.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:55.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:55.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:55.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:55.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:56:55.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:56:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:55.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:56:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:56:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:56:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:55.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:56:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:56:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:56:57.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:57.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:56:57.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:56:57.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:57.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:56:57.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:56:57.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:56:57.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:56:57.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:56:57.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:56:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:56:57.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:56:57.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:56:57.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:56:57.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:56:57.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:56:57.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:57.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:58.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:58.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:56:58.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:58.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:58.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:56 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:56' 2026-02-22T21:56:58.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:58.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:58.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:58.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:58.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:58.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:56' 2026-02-22T21:56:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:56:56' 2026-02-22T21:56:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:58.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:56' 2026-02-22T21:56:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:58.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:56:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:56:56' 2026-02-22T21:56:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:58.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:56' 2026-02-22T21:56:58.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:58.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:58.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:58.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:58.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:58.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:58.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:56:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:56:59.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:56 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:56:59.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:56:56' 2026-02-22T21:56:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:56:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:56:59.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:56:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:56:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:56:59.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:56:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:59.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:56:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:56:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:56:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:56:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:56' 2026-02-22T21:56:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:56:56' 2026-02-22T21:56:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:56:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:56:56' 2026-02-22T21:56:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:56:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:56:59.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:56:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:56:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:56:56' 2026-02-22T21:56:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:56:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:56:59.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:56:56' 2026-02-22T21:56:59.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:56:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:56:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:56:59.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:56:59.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:56:59.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:56:59.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:56:59.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:56:59.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:56:59.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:57:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:00.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:00.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:00.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:57:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:00.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:57:00.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:00.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:00.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:00.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:00.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:57:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:00.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:57:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:02.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:57:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:02.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:02.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:57:02.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:02.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:02.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:02.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:02.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:02.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:57:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:02.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:02.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:57:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:57:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:57:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:57:03.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:57:03.096 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:57:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:57:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:03.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:57:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:57:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:57:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:57:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:03.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:03.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:57:03.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:03.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:57:03.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:03.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:03.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:57:03.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=98, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:03.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=98, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:03.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:03.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=98, entry_tid=0' 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:03.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:03.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:57:03.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=98, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:03.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=98, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:03.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=98, entry_tid=0' 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=98, entry_tid=0' -a 'object_number=0, tag_tid=98, entry_tid=0' = 'object_number=0, tag_tid=98, entry_tid=0' 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:03.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:04.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:04.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:01 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":95},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":98}} 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:01' 2026-02-22T21:57:04.258 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:57:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:57:04.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:57:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:04.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:04.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:04.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:01' 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:57:01' 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:01' 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:04.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:04.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:57:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":95},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":98}}2026-02-22 21:57:01' 2026-02-22T21:57:04.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:04.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:01' 2026-02-22T21:57:04.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:04.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:57:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:57:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:04.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:04.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:05.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:05.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:05.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:05.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:05.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:05.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":95},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":98}} 2026-02-22T21:57:05.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:01 2026-02-22T21:57:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:01' 2026-02-22T21:57:05.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:57:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:57:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:05.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":95},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":98}} 2026-02-22T21:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:05.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:05.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:05.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:01' 2026-02-22T21:57:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:57:01' 2026-02-22T21:57:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:05.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:01' 2026-02-22T21:57:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:05.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":95},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":98}}43184322mirror.0trial1042026-02-22 21:57:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:57:01' 2026-02-22T21:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:01' 2026-02-22T21:57:05.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:57:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:57:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:57:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:57:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:57:05.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:05.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:06.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:06.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:06.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:06.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:06.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:06.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:06.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:06.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:57:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:08.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:08.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:57:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:08.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:08.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:08.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:08.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:08.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:08.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:08.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:08.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:08.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:08.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:08.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:57:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:57:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:09.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:07 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:06' 2026-02-22T21:57:09.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:57:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:09.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:11.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:11.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:11.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:11.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:11.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:11.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:07 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:06' 2026-02-22T21:57:11.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:57:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:11.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:15.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:15.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:11 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:15.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:11' 2026-02-22T21:57:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:15.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:15.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:15.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:15.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:15.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:15.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:15.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:11' 2026-02-22T21:57:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:57:11' 2026-02-22T21:57:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:11' 2026-02-22T21:57:15.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:15.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:15.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:57:11cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:57:11' 2026-02-22T21:57:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:15.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:11' 2026-02-22T21:57:15.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:15.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:15.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:15.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:16.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:16.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:16.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:11 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:11' 2026-02-22T21:57:16.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:16.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:16.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:16.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:16.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:16.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:11' 2026-02-22T21:57:16.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:57:11' 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:11' 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:16.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:16.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:57:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:57:11' 2026-02-22T21:57:16.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:16.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:16.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:16' 2026-02-22T21:57:16.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:16.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:16.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:57:16.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:57:16.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:57:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:57:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:16.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:57:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:57:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:16.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:57:17.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:17.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:17.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:17.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:17.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:17.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:17.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:57:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:57:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:19.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:19.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:19.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:19.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:19.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:19.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:19.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:19.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:19.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:19.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:57:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:57:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:57:23.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:23.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:23.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:23.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:23.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:23.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:23.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:23.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:23.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:24.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:24.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:24.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:24.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:24.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:24.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:57:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:57:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:57:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:57:24.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:24.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:57:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:57:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:57:24.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:57:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:57:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:57:24.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:24.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:24.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:57:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:57:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:24.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:24.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:24.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:24.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:24.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:57:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:24.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:57:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=99, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=99, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:24.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=99, entry_tid=0' 2026-02-22T21:57:24.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:24.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:57:24.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:24.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=99, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=99, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:24.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:24.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=99, entry_tid=0' 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=99, entry_tid=0' -a 'object_number=0, tag_tid=99, entry_tid=0' = 'object_number=0, tag_tid=99, entry_tid=0' 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:24.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:24.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:25.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:25.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:21 2026-02-22T21:57:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":96},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":99}} 2026-02-22T21:57:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:21' 2026-02-22T21:57:25.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:57:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:57:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:25.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:57:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:25.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:25.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:25.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:25.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:25.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:21' 2026-02-22T21:57:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:57:21' 2026-02-22T21:57:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:25.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:21' 2026-02-22T21:57:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:25.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:57:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":96},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":99}}2026-02-22 21:57:21' 2026-02-22T21:57:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:25.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:21' 2026-02-22T21:57:25.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:25.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:57:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:57:25.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:25.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:25.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:26.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:26.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":96},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":99}} 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:21 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:21' 2026-02-22T21:57:26.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:57:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:57:26.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":96},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":99}} 2026-02-22T21:57:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:26.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:26.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:26.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:21' 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:57:21' 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:21' 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:26.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":96},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":99}}45054508mirror.0trial1042026-02-22 21:57:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:57:21' 2026-02-22T21:57:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:26.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:26.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:21' 2026-02-22T21:57:26.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:26.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:57:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:57:26.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:26.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:26.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:57:26.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:57:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:57:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:26.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:26.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:27.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:27.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:27.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:27.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:27.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:27.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:27.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:27.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:27.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:27.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:27.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:27.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:57:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:27.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:57:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:29.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:29.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:29.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:29.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:30.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:57:30.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:30.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:30.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:26 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:28' 2026-02-22T21:57:30.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:30.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:30.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:30.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:30.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:30.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:30.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:30.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:30.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:30.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:30.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:30.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:26' 2026-02-22T21:57:30.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:57:26' 2026-02-22T21:57:30.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:30.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:26' 2026-02-22T21:57:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:30.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:57:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:57:28' 2026-02-22T21:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:30.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:26' 2026-02-22T21:57:30.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:30.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:30.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:30.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:57:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:31.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:31 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:26' 2026-02-22T21:57:31.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:31.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:31.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:31.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:31.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:31.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:31.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:31' 2026-02-22T21:57:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:57:31' 2026-02-22T21:57:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:31.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:31' 2026-02-22T21:57:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:31.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:57:31cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:57:26' 2026-02-22T21:57:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:31.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:31' 2026-02-22T21:57:31.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:31.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:31.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:31.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:57:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:57:31.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:57:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:57:31.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:31.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:32.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:57:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:32.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:57:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:32.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:57:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:32.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:32.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:57:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:32.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:57:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:57:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:32.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:32.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:32.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:32.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:57:32.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:57:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:57:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:57:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:32.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:57:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:57:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:33.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:57:33.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:57:33.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:57:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:57:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:57:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:57:33.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:33.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:33.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:33.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:33.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:33.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:33.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:33.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:57:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:57:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:33.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=103, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:33.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=103, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:33.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:33.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=103, entry_tid=0' 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:57:33.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:57:33.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:33.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=103, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:33.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=103, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:57:33.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=103, entry_tid=0' 2026-02-22T21:57:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=103, entry_tid=0' -a 'object_number=0, tag_tid=103, entry_tid=0' = 'object_number=0, tag_tid=103, entry_tid=0' 2026-02-22T21:57:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:57:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:33.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:33.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:34.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:34.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:31 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:34.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":100},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":103}} 2026-02-22T21:57:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:31' 2026-02-22T21:57:34.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:57:34.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:57:34.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:34.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:57:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:36.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:36.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:36.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:31 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":100},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":103}} 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:31' 2026-02-22T21:57:36.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:57:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:57:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:36.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:57:40.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:40.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:36 2026-02-22T21:57:40.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":100},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":103}} 2026-02-22T21:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:31' 2026-02-22T21:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:57:40.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:57:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:40.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:40.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:40.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:40.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:40.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:40.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:36' 2026-02-22T21:57:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:57:36' 2026-02-22T21:57:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:40.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:36' 2026-02-22T21:57:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:40.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:57:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":100},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":103}}2026-02-22 21:57:31' 2026-02-22T21:57:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:36' 2026-02-22T21:57:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:40.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:57:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:57:40.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:57:40.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:40.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:41.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:57:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:41.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:41.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":100},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":103}} 2026-02-22T21:57:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:41.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:31 2026-02-22T21:57:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:36' 2026-02-22T21:57:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:57:41.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:57:41.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:41.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":100},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":103}} 2026-02-22T21:57:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:41.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:41.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:41.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:31' 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:57:31' 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:31' 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:41.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:41.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":100},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":103}}43184322mirror.0trial1042026-02-22 21:57:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:57:36' 2026-02-22T21:57:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:41.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:31' 2026-02-22T21:57:41.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:57:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:41.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:57:41.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:41.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:57:41.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:57:41.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:41.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:41.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:42.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:57:42.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:42.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:42.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:42.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:42.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:42.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:42.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:42.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:42.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:42.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:42.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:42.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:42.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:42.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:57:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:42.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:44.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:57:44.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:44.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:44.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:44.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:44.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:57:44.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:57:44.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:57:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:57:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:57:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:44.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:57:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:57:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:44.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:45.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:45.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:43 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:45.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:41' 2026-02-22T21:57:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:45.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:57:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:45.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:47.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:47.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:46 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:47.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:46' 2026-02-22T21:57:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:47.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:47.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:47.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:47.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:47.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:46' 2026-02-22T21:57:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:57:46' 2026-02-22T21:57:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:47.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:46' 2026-02-22T21:57:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:47.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:57:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:57:46' 2026-02-22T21:57:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:47.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:47.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:46' 2026-02-22T21:57:47.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:47.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:47.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:47.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:57:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:47.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:47.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:48.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:57:48.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:48.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:48.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:48.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:46 2026-02-22T21:57:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:46' 2026-02-22T21:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:48.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:48.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:48.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:48.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:48.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:46' 2026-02-22T21:57:48.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:57:46' 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:46' 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:48.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:57:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:57:46' 2026-02-22T21:57:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:48.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:46' 2026-02-22T21:57:48.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:48.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:57:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:57:48.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:57:48.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:48.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:57:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:48.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:57:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:49.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:49.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:49.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:49.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:49.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:49.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:49.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:49.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:57:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:49.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:57:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:57:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:51.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:51.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:51.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:51.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:51.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:51.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:51.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:51.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:51.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:51.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:51.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:57:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:57:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:57:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:57:52.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:52.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:57:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:57:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:57:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:57:52.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:57:52.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:52.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:52.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:57:52.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:52.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:57:52.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:52.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=104, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:52.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=104, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:52.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:52.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=104, entry_tid=0' 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:57:52.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:57:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:57:52.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:52.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:57:52.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:57:52.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:57:52.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=104, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:52.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=104, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:57:52.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=104, entry_tid=0' 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=104, entry_tid=0' -a 'object_number=0, tag_tid=104, entry_tid=0' = 'object_number=0, tag_tid=104, entry_tid=0' 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:52.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:52.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:57:53.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:53.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:53.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:53.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:53.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:53.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:51 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":101},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":104}} 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:51' 2026-02-22T21:57:53.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:57:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:57:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:53.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:57:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:53.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:53.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:53.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:53.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:53.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:51' 2026-02-22T21:57:53.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:57:51' 2026-02-22T21:57:53.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:53.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:57:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:51' 2026-02-22T21:57:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:53.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:53.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:57:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":101},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":104}}2026-02-22 21:57:51' 2026-02-22T21:57:53.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:53.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:51' 2026-02-22T21:57:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:57:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:57:53.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:57:53.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:53.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:53.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:53.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":101},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":104}} 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:51 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:54.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:51' 2026-02-22T21:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:57:54.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:54.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":101},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":104}} 2026-02-22T21:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:54.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:54.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:54.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:54.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:51' 2026-02-22T21:57:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:57:51' 2026-02-22T21:57:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:54.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:57:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:51' 2026-02-22T21:57:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:54.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":101},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":104}}45054508mirror.0trial1042026-02-22 21:57:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:57:51' 2026-02-22T21:57:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:54.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:54.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:51' 2026-02-22T21:57:54.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:54.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:57:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:57:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:57:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:57:54.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:57:54.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:54.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:54.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:55.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:55.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:55.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:55.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:55.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:55.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:55.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:55.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:55.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:55.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:55.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:55.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:57:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:57:55.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:55.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:57:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:57:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:57:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:55.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:57:57.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:57:57.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:57:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:57:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:57:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:57:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:57:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:57.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:57:57.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:57:57.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:57:57.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:57:57.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:57:57.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:57:57.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:57:57.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:57:57.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:57:57.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:57:57.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:57:57.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:57:57.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:57:57.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:57.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:57.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:57:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:58.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:58.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:56 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:58.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:57:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:56' 2026-02-22T21:57:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:58.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:58.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:58.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:58.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:58.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:56' 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:57:56' 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:56' 2026-02-22T21:57:58.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:58.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:57:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:57:56' 2026-02-22T21:57:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:58.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:58.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:56' 2026-02-22T21:57:58.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:58.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:58.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:58.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:58.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:58.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:57:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:56 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:57:56' 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:57:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:57:59.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:57:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:57:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:57:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:57:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:57:59.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:57:59.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:57:59.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:57:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:56' 2026-02-22T21:57:59.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:57:56' 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:57:56' 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:57:59.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:57:59.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:57:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:57:56' 2026-02-22T21:57:59.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:57:59.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:57:59.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:57:56' 2026-02-22T21:57:59.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:57:59.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:57:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:57:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:57:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:57:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:57:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:57:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:57:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:57:59.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:57:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:57:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:57:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:58:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:00.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:00.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:00.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:00.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:00.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:00.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:00.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:58:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:00.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:02.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:58:02.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:02.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:02.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:02.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:02.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:58:02.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:02.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:02.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:02.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:02.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:02.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:02.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:58:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:58:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:58:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:02.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:58:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:58:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:03.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:03.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:58:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:03.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:58:03.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:03.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:03.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:03.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:03.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:58:03.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:58:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:03.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:58:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:58:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:03.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:03.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:58:03.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:03.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=108, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=108, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:03.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:03.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=108, entry_tid=0' 2026-02-22T21:58:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:58:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:03.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:58:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:58:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:03.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:58:03.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=108, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:03.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=108, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:03.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=108, entry_tid=0' 2026-02-22T21:58:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=108, entry_tid=0' -a 'object_number=0, tag_tid=108, entry_tid=0' = 'object_number=0, tag_tid=108, entry_tid=0' 2026-02-22T21:58:03.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:03.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:04.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:04.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:01 2026-02-22T21:58:04.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":105},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":108}} 2026-02-22T21:58:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:01' 2026-02-22T21:58:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:58:04.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:58:04.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:04.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:58:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:04.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:04.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:04.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:01' 2026-02-22T21:58:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:58:01' 2026-02-22T21:58:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:04.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:01' 2026-02-22T21:58:04.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:04.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:04.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:58:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":105},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":108}}2026-02-22 21:58:01' 2026-02-22T21:58:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:04.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:01' 2026-02-22T21:58:04.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:04.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:58:04.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:04.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:58:04.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:58:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:05.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":105},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":108}} 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:01 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:05.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:01' 2026-02-22T21:58:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:58:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:58:05.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":105},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":108}} 2026-02-22T21:58:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:05.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:05.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:05.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:01' 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:58:01' 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:01' 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:05.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:05.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":105},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":108}}43184322mirror.0trial1042026-02-22 21:58:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:58:01' 2026-02-22T21:58:05.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:05.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:05.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:01' 2026-02-22T21:58:05.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:05.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:58:05.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:05.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:58:05.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:58:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:58:05.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:58:05.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:58:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:58:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:06.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:58:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:06.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:06.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:58:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:06.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:06.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:06.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:06.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:06.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:06.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:06.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:58:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:06.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:08.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:58:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:08.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:08.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:08.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:08.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:08.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:08.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:08.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:08.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:08.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:58:08.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:08.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:09.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:09.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:09.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:58:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:07 2026-02-22T21:58:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:09.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:06' 2026-02-22T21:58:09.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:58:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:09.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:11.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:58:11.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:11.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:11.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:11.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:11.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:07 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:06' 2026-02-22T21:58:11.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:58:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:11.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:58:15.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:15.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:15.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:15.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:11 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:11' 2026-02-22T21:58:15.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:15.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:11' 2026-02-22T21:58:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:58:11' 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:11' 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:58:11cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:58:11' 2026-02-22T21:58:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:11' 2026-02-22T21:58:15.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:58:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:15.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:58:15.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:15.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:15.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:16.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:16.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:16.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:16 2026-02-22T21:58:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:11' 2026-02-22T21:58:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:16.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:16.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:16.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:16' 2026-02-22T21:58:16.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:58:16' 2026-02-22T21:58:16.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:16.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:16.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:16.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:16.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:16' 2026-02-22T21:58:16.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:16.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:16.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:58:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:58:11' 2026-02-22T21:58:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:16' 2026-02-22T21:58:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:58:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:58:16.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:58:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:58:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:58:16.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:58:16.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:58:16.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:16.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:17.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:58:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:17.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:17.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:17.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:17.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:17.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:17.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:17.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:17.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:17.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:58:17.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:17.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:58:19.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:19.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:19.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:19.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:19.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:19.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:19.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:20.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:20.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:58:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:20.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:58:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:58:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:24.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:24.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:24.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:24.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:24.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:24.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:24.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:24.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:24.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:24.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:24.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:24.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:24.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:24.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:24.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:58:24.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:24.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:24.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:58:24.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:24.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:58:24.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=109, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:24.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=109, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:24.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:24.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=109, entry_tid=0' 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:24.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:24.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:58:24.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:24.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:24.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:24.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:24.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=109, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:24.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=109, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:24.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=109, entry_tid=0' 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=109, entry_tid=0' -a 'object_number=0, tag_tid=109, entry_tid=0' = 'object_number=0, tag_tid=109, entry_tid=0' 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:24.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:25.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:25.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:21 2026-02-22T21:58:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:25.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":106},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":109}} 2026-02-22T21:58:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:21' 2026-02-22T21:58:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:58:25.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:58:25.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:25.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:58:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:25.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:25.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:25.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:21' 2026-02-22T21:58:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:58:21' 2026-02-22T21:58:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:25.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:21' 2026-02-22T21:58:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:25.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:58:21cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":106},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":109}}2026-02-22 21:58:21' 2026-02-22T21:58:25.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:25.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:25.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:21' 2026-02-22T21:58:25.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:58:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:25.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:58:25.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:25.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:25.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:26.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":106},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":109}} 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:21 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:21' 2026-02-22T21:58:26.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:58:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:58:26.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":106},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":109}} 2026-02-22T21:58:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:26.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:26.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:26.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:26.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:21' 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:58:21' 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:21' 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:26.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":106},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":109}}45054508mirror.0trial1042026-02-22 21:58:21cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:58:21' 2026-02-22T21:58:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:26.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:21' 2026-02-22T21:58:26.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:58:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:26.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:58:26.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:26.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:58:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:27.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:27.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:27.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:27.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:27.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:27.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:27.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:27.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:27.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:27.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:27.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:27.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:58:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:27.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:29.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:29.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:29.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:29.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:29.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:29.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:29.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:29.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:29.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:29.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:29.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:58:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:30.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:26 2026-02-22T21:58:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:30.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:28' 2026-02-22T21:58:30.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:30.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:30.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:30.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:30.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:30.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:30.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:26' 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:58:26' 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:26' 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:30.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:30.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:58:26cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedstopped2026-02-22 21:58:28' 2026-02-22T21:58:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:30.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:26' 2026-02-22T21:58:30.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:30.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:30.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:58:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:58:30.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:30.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:58:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:31.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:31.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:31 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:31.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:26' 2026-02-22T21:58:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:31.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:31.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:31.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:31.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:31.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:31.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:31.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:31.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:31.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:31.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:31' 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:58:31' 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:31' 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:31.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:58:31cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:58:26' 2026-02-22T21:58:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:31.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:31.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:31' 2026-02-22T21:58:31.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:58:31.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:58:31.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:58:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:31.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:58:31.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:58:31.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:58:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:58:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:32.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:58:32.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:58:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:32.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:32.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:32.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:32.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:58:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:58:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:58:33.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:58:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:58:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:58:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:58:33.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:33.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:58:33.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:58:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:58:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:33.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:33.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:33.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:58:33.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:58:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:33.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:33.249 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:33.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:33.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:58:33.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:33.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=113, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:33.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=113, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=113, entry_tid=0' 2026-02-22T21:58:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:58:33.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:58:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:33.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:58:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:58:33.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:33.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=113, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=113, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:58:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=113, entry_tid=0' 2026-02-22T21:58:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=113, entry_tid=0' -a 'object_number=0, tag_tid=113, entry_tid=0' = 'object_number=0, tag_tid=113, entry_tid=0' 2026-02-22T21:58:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:58:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:33.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:34.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:34.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:34.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:31 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":110},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":113}} 2026-02-22T21:58:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:31' 2026-02-22T21:58:34.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:58:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:58:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:34.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:58:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:31 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:36.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:36.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:36.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":110},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":113}} 2026-02-22T21:58:36.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:31' 2026-02-22T21:58:36.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:58:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:58:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:40.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:40.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:40.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:40.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:36 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":110},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":113}} 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:31' 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:58:40.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:58:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:40.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:40.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:40.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:40.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:40.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:36' 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:58:36' 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:36' 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:40.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:58:36cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":110},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":113}}2026-02-22 21:58:31' 2026-02-22T21:58:40.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:40.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:36' 2026-02-22T21:58:40.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:40.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:58:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:58:40.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:40.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:41.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:41.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:41.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":110},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":113}} 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:31 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:36' 2026-02-22T21:58:41.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:58:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:58:41.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:41.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":110},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":113}} 2026-02-22T21:58:41.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:41.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:41.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:41.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:41.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:31' 2026-02-22T21:58:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:58:31' 2026-02-22T21:58:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:41.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:31' 2026-02-22T21:58:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:41.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:41.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":110},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":113}}43184322mirror.0trial1042026-02-22 21:58:31cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:58:36' 2026-02-22T21:58:41.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:41.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:31' 2026-02-22T21:58:41.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:41.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:58:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:58:41.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:41.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T21:58:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test 2026-02-22T21:58:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:41.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test 2026-02-22T21:58:41.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:58:41.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:58:41.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:41.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:58:42.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:42.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:58:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:42.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:42.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:42.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:42.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:42.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:42.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:42.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:42.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:42.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:42.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:42.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:42.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:58:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:42.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T21:58:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:44.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:44.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:58:44.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:44.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:44.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:44.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:58:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:44.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:44.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:44.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:44.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:45.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:43 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:41' 2026-02-22T21:58:45.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:58:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:45.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:47.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:47.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:46 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:46' 2026-02-22T21:58:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:47.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:47.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:47.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:47.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:47.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:47.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:47.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:47.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:46' 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:58:46' 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:46' 2026-02-22T21:58:47.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:47.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:47.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:58:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:58:46' 2026-02-22T21:58:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:47.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:46' 2026-02-22T21:58:47.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:47.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:58:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:47.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:58:47.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:47.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:47.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:48.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:48.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:48.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:46 2026-02-22T21:58:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:46' 2026-02-22T21:58:48.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:48.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:48.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:48.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:48.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:46' 2026-02-22T21:58:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:58:46' 2026-02-22T21:58:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:46' 2026-02-22T21:58:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:48.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:58:46cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:58:46' 2026-02-22T21:58:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:46' 2026-02-22T21:58:48.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:58:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:58:48.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:48.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test 2026-02-22T21:58:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:58:48.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test 2026-02-22T21:58:48.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test 2026-02-22T21:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test started 2026-02-22T21:58:48.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:48.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:58:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:49.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:49.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:49.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:49.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:49.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:49.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:49.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:49.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:49.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:49.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:49.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:49.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:50.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:50.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:50.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:58:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:50.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test started 2026-02-22T21:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:52.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:52.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:52.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:52.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:52.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:52.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:52.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:52.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:52.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:52.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:52.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror test 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster2 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:52.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:58:52.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:58:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster2 mirror test 2026-02-22T21:58:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:52.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/test 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:52.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:58:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:52.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:58:52.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/test 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:58:52.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:58:52.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:52.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:52.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:58:52.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster1 mirror test 2026-02-22T21:58:52.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '' 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:52.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:58:52.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=114, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:52.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=114, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:52.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:52.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=114, entry_tid=0' 2026-02-22T21:58:52.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster1 mirror test 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster1 mirror test '..*' 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:58:52.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:58:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 journal status --image mirror/test 2026-02-22T21:58:52.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:52.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:58:52.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:58:52.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:58:52.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=114, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:52.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=605c498e-0476-4c03-b186-dac6ca3f245b, commit_position=[positions=[[object_number=0, tag_tid=114, entry_tid=0], [object_number=1, tag_tid=13, entry_tid=101], [object_number=3, tag_tid=13, entry_tid=99], [object_number=2, tag_tid=13, entry_tid=98]]], state=connected] 2026-02-22T21:58:52.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:58:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=114, entry_tid=0' 2026-02-22T21:58:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=114, entry_tid=0' -a 'object_number=0, tag_tid=114, entry_tid=0' = 'object_number=0, tag_tid=114, entry_tid=0' 2026-02-22T21:58:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+stopped 2026-02-22T21:58:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:52.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+stopped '' '' 2026-02-22T21:58:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:53.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:53.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:53.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:51 2026-02-22T21:58:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":111},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":114}} 2026-02-22T21:58:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:51' 2026-02-22T21:58:53.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:58:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:58:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:53.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:58:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:53.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:53.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:53.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:51' 2026-02-22T21:58:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+stopped '2026-02-22 21:58:51' 2026-02-22T21:58:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:53.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:58:53.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:51' 2026-02-22T21:58:53.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:53.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:53.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary43184322mirror.0trial1042026-02-22 21:58:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":111},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":114}}2026-02-22 21:58:51' 2026-02-22T21:58:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:53.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:53.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:51' 2026-02-22T21:58:53.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:53.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:58:53.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:58:53.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:58:53.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:58:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:53.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+replaying 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:53.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:54.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+replaying '' '' 2026-02-22T21:58:54.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:58:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":111},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":114}} 2026-02-22T21:58:54.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:51 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:51' 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:58:54.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:58:54.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:54.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":111},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":114}} 2026-02-22T21:58:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:54.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:51' 2026-02-22T21:58:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+replaying '2026-02-22 21:58:51' 2026-02-22T21:58:54.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:58:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:51' 2026-02-22T21:58:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:54.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":111},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":114}}45054508mirror.0trial1042026-02-22 21:58:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:58:51' 2026-02-22T21:58:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:51' 2026-02-22T21:58:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:58:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:58:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:58:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:58:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror test 2026-02-22T21:58:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:54.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/test 2026-02-22T21:58:54.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T21:58:54.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test 2026-02-22T21:58:54.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:54.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test stopped 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:54.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:55.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:55.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:55.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:55.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:55.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:55.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:55.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:55.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:58:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:58:55.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:58:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:58:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = started 2026-02-22T21:58:55.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:58:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test stopped 2026-02-22T21:58:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:57.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:58:57.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:58:57.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:58:57.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:58:57.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:58:57.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:58:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:58:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:58:57.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:58:57.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:57.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:57.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:58:57.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:58:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:58:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:58:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:58:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:58:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:58:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+unknown 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+unknown '' '' 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:58.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:58:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:58.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:56 2026-02-22T21:58:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:58:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:56' 2026-02-22T21:58:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:58.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:58.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:58.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:58.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:58.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:58.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:58.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:58.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:58.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:56' 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+unknown '2026-02-22 21:58:56' 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:56' 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:58.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:58:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary43184322mirror.0trial1042026-02-22 21:58:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 21:58:56' 2026-02-22T21:58:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:58:58.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:58:58.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:56' 2026-02-22T21:58:58.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:58:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:58:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:58.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:58:58.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:58:58.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+unknown 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:58:58.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:58:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+unknown '' '' 2026-02-22T21:58:59.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:59.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:58:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:58:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:58:59.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:56 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:58:56' 2026-02-22T21:58:59.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T21:58:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T21:58:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:58:59.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T21:58:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:58:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:58:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:58:59.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:58:59.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:58:59.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:58:59.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:58:59.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:58:59.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:56' 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+unknown '2026-02-22 21:58:56' 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:58:56' 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:58:59.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 21:58:56cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+unknownremote image is not primary2026-02-22 21:58:56' 2026-02-22T21:59:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:00.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:59:00.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:58:56' 2026-02-22T21:59:00.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:59:00.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T21:59:00.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T21:59:00.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T21:59:00.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T21:59:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:00.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster2 mirror test 2026-02-22T21:59:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T21:59:00.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image promote mirror/test 2026-02-22T21:59:00.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:59:00.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:00.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:01.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:59:01.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:01.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:01.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:01.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:01.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:01.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:59:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:59:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:59:01.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T21:59:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:59:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T21:59:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:59:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:03.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:03.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T21:59:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:03.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:03.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:03.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T21:59:03.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:03.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:03.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test 2026-02-22T21:59:03.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:03.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test' 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test 2026-02-22T21:59:03.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:03.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:59:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test 2026-02-22T21:59:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:59:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:59:03.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:03.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:59:03.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:03.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:03.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:03.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test 2026-02-22T21:59:03.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test 2026-02-22T21:59:03.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '' 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:59:03.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:59:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:59:03.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:59:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:03.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=118, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:59:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=118, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:59:03.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=0, tag_tid=118, entry_tid=0' 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test '..*' 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:03.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T21:59:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:59:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test.status 2026-02-22T21:59:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test.status 2026-02-22T21:59:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:03.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:59:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test 2026-02-22T21:59:03.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:59:03.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:03.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:03.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:03.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=0, tag_tid=118, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:59:03.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=0, tag_tid=118, entry_tid=0], [object_number=1, tag_tid=18, entry_tid=101], [object_number=3, tag_tid=18, entry_tid=99], [object_number=2, tag_tid=18, entry_tid=98]]], state=connected] 2026-02-22T21:59:03.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.status 2026-02-22T21:59:03.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=0, tag_tid=118, entry_tid=0' 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=0, tag_tid=118, entry_tid=0' -a 'object_number=0, tag_tid=118, entry_tid=0' = 'object_number=0, tag_tid=118, entry_tid=0' 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test up+stopped 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:03.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:04.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test up+stopped '' '' 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:04.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:01 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:04.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:59:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:59:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}} 2026-02-22T21:59:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:01' 2026-02-22T21:59:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:59:04.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:59:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:59:04.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:59:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:04.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:04.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:01' 2026-02-22T21:59:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test up+stopped '2026-02-22 21:59:01' 2026-02-22T21:59:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:04.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:01' 2026-02-22T21:59:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:04.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:59:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}2026-02-22 21:59:01' 2026-02-22T21:59:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:04.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:59:04.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:01' 2026-02-22T21:59:04.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:59:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:59:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:59:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:04.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:04.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying '' '' 2026-02-22T21:59:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T21:59:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T21:59:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: a4e8f094-9fd1-42cf-992f-29d2efd77535 2026-02-22T21:59:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:59:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}} 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:01 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:01' 2026-02-22T21:59:05.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:59:05.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:59:05.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:59:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}} 2026-02-22T21:59:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:05.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:01' 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 21:59:01' 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:05.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:01' 2026-02-22T21:59:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:05.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}43184322mirror.0trial1042026-02-22 21:59:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:59:01' 2026-02-22T21:59:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T21:59:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:01' 2026-02-22T21:59:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T21:59:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:59:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:59:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T21:59:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: force promote' 2026-02-22T21:59:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:59:05.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:59:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:59:05 TEST: force promote 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:59:05 TEST: force promote 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ force_promote_image=test_force_promote 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote journal 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:59:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test_force_promote 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:05.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test_force_promote 2026-02-22T21:59:05.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:59:05.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:59:05.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:59:05.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:59:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:59:05.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 5555.67 bytes/sec: 22 MiB/s 2026-02-22T21:59:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-22T21:59:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T21:59:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:59:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:05.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T21:59:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:59:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:59:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:06.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:59:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:06.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:06.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:1 2026-02-22T21:59:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:1 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:1 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:06.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T21:59:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:59:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:59:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:06.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T21:59:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T21:59:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T21:59:06.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T21:59:06.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:06.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T21:59:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T21:59:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T21:59:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T21:59:06.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:06.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:3 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T21:59:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:3 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:3 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T21:59:06.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:59:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:59:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:06.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T21:59:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T21:59:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T21:59:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T21:59:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:59:08.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:59:08.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:59:08.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:08.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:08.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:59:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:08.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:08.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:08.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:08.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:59:08.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:59:08.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:59:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote 2026-02-22T21:59:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:08.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote started 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:08.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_force_promote started 2026-02-22T21:59:09.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:59:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:09.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:09.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:09.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:09.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:09.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:09.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:09.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:09.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:09.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:09.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:09.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:09.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:09.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:09.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:09.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:09.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:09.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:09.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:09.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:09.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:09.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:09.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:09.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:09.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:09.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:09.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:09.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:09.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote'] 2026-02-22T21:59:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:09.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:09.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:59:10.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:10.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:10.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:10.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:10.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:10.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:10.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:10.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:10.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:10.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:10.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:10.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:10.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_force_promote 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:10.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test_force_promote 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:10.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_force_promote 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_force_promote ']' 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_force_promote' 2026-02-22T21:59:10.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_force_promote 2026-02-22T21:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:10.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_force_promote 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:59:10.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:10.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:10.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:10.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_force_promote 2026-02-22T21:59:10.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_force_promote 2026-02-22T21:59:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote 2026-02-22T21:59:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_force_promote '' 2026-02-22T21:59:10.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote 2026-02-22T21:59:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:59:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_force_promote 2026-02-22T21:59:10.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:10.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:10.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:10.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_force_promote 2026-02-22T21:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote 2026-02-22T21:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_force_promote '..*' 2026-02-22T21:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:10.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote 2026-02-22T21:59:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:59:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_force_promote 2026-02-22T21:59:10.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:10.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote.status 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:10.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+replaying primary_position 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:10.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+replaying primary_position '' 2026-02-22T21:59:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:59:11.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:11.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:11.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:11.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-22T21:59:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote: 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: aa485948-382e-4a5a-9a9a-c84fb42aa8b6 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:09 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:11' 2026-02-22T21:59:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:59:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:59:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:59:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:59:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:11.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:11.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:11.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:09' 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+replaying '2026-02-22 21:59:09' 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:09' 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:11.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:11.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}43184322mirror.0trial1042026-02-22 21:59:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:59:11test_force_promoteaa485948-382e-4a5a-9a9a-c84fb42aa8b6up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:59:09cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:59:11' 2026-02-22T21:59:11.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:11.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T21:59:11.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:09' 2026-02-22T21:59:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T21:59:11.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:59:11.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:11.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:59:11.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-22T21:59:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:11.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:11.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-22T21:59:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:12.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:12.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-22T21:59:12.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote: 2026-02-22T21:59:12.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: aa485948-382e-4a5a-9a9a-c84fb42aa8b6 2026-02-22T21:59:12.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:12.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:12.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:12.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:11 2026-02-22T21:59:12.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:12.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:59:12.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:59:12.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:59:12.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:09' 2026-02-22T21:59:12.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:59:12.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:59:12.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:59:12.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:59:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:12.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:12.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:12.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:12.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:12.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:11' 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-22 21:59:11' 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:11' 2026-02-22T21:59:12.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:12.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK245054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:59:11cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}2026-02-22 21:59:01test_force_promoteaa485948-382e-4a5a-9a9a-c84fb42aa8b6up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:59:11cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}2026-02-22 21:59:09' 2026-02-22T21:59:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:12.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T21:59:12.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:11' 2026-02-22T21:59:12.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T21:59:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:59:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:12.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:59:12.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test_force_promote --force 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force=--force 2026-02-22T21:59:12.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote --force 2026-02-22T21:59:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:12.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:59:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:12.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:13.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_force_promote stopped 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:13.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:13.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:13.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:13.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:13.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:13.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:13.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:13.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:13.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:59:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:59:13.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:13.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T21:59:13.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:13.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test_force_promote stopped 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:14.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:14.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T21:59:14.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:14.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:14.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T21:59:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T21:59:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T21:59:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:14.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T21:59:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:14.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T21:59:14.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote 2026-02-22T21:59:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T21:59:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T21:59:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote up+stopped 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:15.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote up+stopped '' '' 2026-02-22T21:59:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:16.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote 2026-02-22T21:59:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote: 2026-02-22T21:59:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: aa485948-382e-4a5a-9a9a-c84fb42aa8b6 2026-02-22T21:59:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T21:59:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:16.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:12 2026-02-22T21:59:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:59:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:11' 2026-02-22T21:59:16.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:59:16.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:59:16.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:59:16.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: stopped 2026-02-22T21:59:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:16.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:16.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:16.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:12' 2026-02-22T21:59:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote up+stopped '2026-02-22 21:59:12' 2026-02-22T21:59:16.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:12' 2026-02-22T21:59:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:16.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:16.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK1143184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":115},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}43184322mirror.0trial1042026-02-22 21:59:01cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:59:11test_force_promoteaa485948-382e-4a5a-9a9a-c84fb42aa8b6up+stoppedstopped43184322mirror.0trial1042026-02-22 21:59:12cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:59:11' 2026-02-22T21:59:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:16.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T21:59:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:12' 2026-02-22T21:59:16.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T21:59:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:59:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:16.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:59:16.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote up+stopped 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:16.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:16.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote up+stopped '' '' 2026-02-22T21:59:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T21:59:17.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:17.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote: 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: aa485948-382e-4a5a-9a9a-c84fb42aa8b6 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:16 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T21:59:17.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:16' 2026-02-22T21:59:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T21:59:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T21:59:17.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T21:59:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T21:59:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:17.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:17.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:17.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:16' 2026-02-22T21:59:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote up+stopped '2026-02-22 21:59:16' 2026-02-22T21:59:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T21:59:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:16' 2026-02-22T21:59:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:17.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:17.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK1145054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:59:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":2.4,"entries_behind_primary":0,"entries_per_second":0.1,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}2026-02-22 21:59:16test_force_promoteaa485948-382e-4a5a-9a9a-c84fb42aa8b6up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 21:59:16cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 21:59:16' 2026-02-22T21:59:17.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/last_update' 2026-02-22T21:59:17.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:16' 2026-02-22T21:59:17.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote'\'']/state' 2026-02-22T21:59:17.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T21:59:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T21:59:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T21:59:17.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T21:59:17.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:17.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:17.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster1 mirror test_force_promote 100 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:59:17.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:59:17.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:59:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:59:17.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T21:59:17.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_force_promote 100 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:59:17.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:59:17.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:59:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:59:17.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7143 bytes/sec: 28 MiB/s 2026-02-22T21:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster1 mirror test_force_promote 2026-02-22T21:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:59:17.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster1 mirror test_force_promote 2026-02-22T21:59:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote 2026-02-22T21:59:17.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote 2026-02-22T21:59:17.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:59:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test_force_promote 2026-02-22T21:59:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:59:17.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_force_promote 2026-02-22T21:59:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote 2026-02-22T21:59:17.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote 2026-02-22T21:59:17.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote 2026-02-22T21:59:17.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:59:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: cloned images' 2026-02-22T21:59:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:59:17.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:59:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:59:17 TEST: cloned images 2026-02-22T21:59:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:59:17 TEST: cloned images 2026-02-22T21:59:17.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - default' 2026-02-22T21:59:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:59:17.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:59:17 - default 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:59:17 - default 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ parent_image=test_parent 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ parent_snap=snap 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent test_parent journal 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:17.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror_parent test_parent 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:17.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T21:59:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:17.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/test_parent 2026-02-22T21:59:17.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-22T21:59:17.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:17.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:59:17.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:59:17.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ enable_mirror cluster2 mirror_parent test_parent journal 2026-02-22T21:59:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:59:17.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/test_parent journal 2026-02-22T21:59:17.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Mirroring enabled 2026-02-22T21:59:17.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 info mirror_parent/test_parent 2026-02-22T21:59:17.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test_parent': 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 0 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 15d29fd2facc 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.15d29fd2facc 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T21:59:17.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 21:59:17 2026 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 21:59:17 2026 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 21:59:17 2026 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 15d29fd2facc 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: aa246585-a2e6-4705-9011-effcb6e76c43 2026-02-22T21:59:17.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: true 2026-02-22T21:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror_parent test_parent 100 2026-02-22T21:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:59:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:59:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:59:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:59:17.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror_parent/test_parent --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:59:17.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:59:17.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:59:17.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T21:59:17.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T21:59:17.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:17.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:17.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:17.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap 2026-02-22T21:59:17.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap 2026-02-22T21:59:18.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T21:59:18.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ protect_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T21:59:18.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap 2026-02-22T21:59:18.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap protect mirror_parent/test_parent@snap 2026-02-22T21:59:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image=test_clone 2026-02-22T21:59:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone 2026-02-22T21:59:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T21:59:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T21:59:18.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap 2026-02-22T21:59:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T21:59:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone 2026-02-22T21:59:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T21:59:18.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone --image-feature layering,exclusive-lock,journaling 2026-02-22T21:59:18.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_clone 100 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T21:59:18.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_clone --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T21:59:18.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T21:59:18.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T21:59:18.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 5555.67 bytes/sec: 22 MiB/s 2026-02-22T21:59:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T21:59:18.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ enable_mirror cluster2 mirror_parent test_parent journal 2026-02-22T21:59:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:59:18.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/test_parent journal 2026-02-22T21:59:18.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Mirroring enabled 2026-02-22T21:59:18.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 info mirror_parent/test_parent 2026-02-22T21:59:18.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test_parent': 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 1 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 15d29fd2facc 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.15d29fd2facc 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 21:59:17 2026 2026-02-22T21:59:18.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 21:59:17 2026 2026-02-22T21:59:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 21:59:17 2026 2026-02-22T21:59:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 15d29fd2facc 2026-02-22T21:59:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T21:59:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T21:59:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: aa246585-a2e6-4705-9011-effcb6e76c43 2026-02-22T21:59:18.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: true 2026-02-22T21:59:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror_parent test_parent 2026-02-22T21:59:18.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror_parent test_parent started 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:18.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror_parent test_parent started 2026-02-22T21:59:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:59:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:19.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:19.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:19.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:19.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:19.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:19.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:19.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:19.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:19.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:19.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:19.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:19.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:19.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:19.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:19.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:19.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:19.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:19.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:19.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:19.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:19.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:19.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:19.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:19.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:19.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:19.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:19.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:19.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:19.695 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:19.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:19.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:19.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:19.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:19.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:19.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:19.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:19.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:19.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:59:20.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:20.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:20.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:20.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:20.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:20.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:20.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:20.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:20.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:20.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:20.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:20.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:20.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:20.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:20.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:20.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:20.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:20.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:20.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:20.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:20.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:20.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:20.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:20.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:20.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:20.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:20.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:20.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:20.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror_parent/test_parent'] 2026-02-22T21:59:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:20.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:20.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:20.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T21:59:22.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:22.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:22.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:22.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:22.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:22.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/test_parent 2026-02-22T21:59:23.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:23.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:23.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:59:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:59:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:59:23.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent test_parent 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror_parent test_parent 2026-02-22T21:59:23.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:23.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:59:23.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror_parent test_parent 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_parent ']' 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror_parent/test_parent' 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror_parent/test_parent 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:59:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror_parent/test_parent 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:23.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:59:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:59:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:23.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:23.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:23.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror_parent/test_parent 2026-02-22T21:59:23.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror_parent test_parent 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror_parent test_parent '' 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:59:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/test_parent 2026-02-22T21:59:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=4, entry_tid=3], [object_number=2, tag_tid=4, entry_tid=2], [object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0]]], state=connected] 2026-02-22T21:59:23.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=30345cf1-b49e-44c4-92c9-701ac0512680, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=99], [object_number=2, tag_tid=1, entry_tid=98]]], state=connected] 2026-02-22T21:59:23.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=4, entry_tid=3' 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror_parent test_parent 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror_parent test_parent '..*' 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/test_parent 2026-02-22T21:59:23.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=4, entry_tid=3], [object_number=2, tag_tid=4, entry_tid=2], [object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0]]], state=connected] 2026-02-22T21:59:23.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=30345cf1-b49e-44c4-92c9-701ac0512680, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=99], [object_number=2, tag_tid=1, entry_tid=98]]], state=connected] 2026-02-22T21:59:23.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-02-22T21:59:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=4, entry_tid=3' -a 'object_number=3, tag_tid=4, entry_tid=3' = 'object_number=1, tag_tid=3, entry_tid=1' 2026-02-22T21:59:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' '!=' '' 2026-02-22T21:59:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T21:59:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=1' = '' 2026-02-22T21:59:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=3, entry_tid=1' 2026-02-22T21:59:23.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=4, entry_tid=3' 2026-02-22T21:59:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:59:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:59:23.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=4 2026-02-22T21:59:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-02-22T21:59:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T21:59:23.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:59:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=3 2026-02-22T21:59:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=4, entry_tid=3' 2026-02-22T21:59:23.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:59:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:59:23.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=3 2026-02-22T21:59:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=1' 2026-02-22T21:59:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T21:59:23.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T21:59:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=1 2026-02-22T21:59:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 4 = 3 -a 3 -le 1 2026-02-22T21:59:23.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:59:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:23.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:59:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror_parent test_parent 2026-02-22T21:59:23.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_parent ']' 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror_parent/test_parent' 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror_parent/test_parent 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:59:23.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:23.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror_parent/test_parent 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:59:23.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:23.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:23.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror_parent/test_parent 2026-02-22T21:59:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror_parent test_parent 2026-02-22T21:59:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror_parent test_parent '' 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.638 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:23.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/test_parent 2026-02-22T21:59:23.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:23.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:23.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:23.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=4, entry_tid=3], [object_number=2, tag_tid=4, entry_tid=2], [object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0]]], state=connected] 2026-02-22T21:59:23.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=30345cf1-b49e-44c4-92c9-701ac0512680, commit_position=[positions=[[object_number=3, tag_tid=4, entry_tid=3], [object_number=2, tag_tid=4, entry_tid=2], [object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0]]], state=connected] 2026-02-22T21:59:23.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=4, entry_tid=3' 2026-02-22T21:59:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror_parent test_parent 2026-02-22T21:59:23.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror_parent test_parent '..*' 2026-02-22T21:59:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror_parent 2026-02-22T21:59:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_parent 2026-02-22T21:59:23.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:59:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/test_parent 2026-02-22T21:59:23.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=4, entry_tid=3], [object_number=2, tag_tid=4, entry_tid=2], [object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0]]], state=connected] 2026-02-22T21:59:23.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=30345cf1-b49e-44c4-92c9-701ac0512680, commit_position=[positions=[[object_number=3, tag_tid=4, entry_tid=3], [object_number=2, tag_tid=4, entry_tid=2], [object_number=1, tag_tid=4, entry_tid=1], [object_number=0, tag_tid=4, entry_tid=0]]], state=connected] 2026-02-22T21:59:23.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.status 2026-02-22T21:59:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=4, entry_tid=3' 2026-02-22T21:59:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=4, entry_tid=3' -a 'object_number=3, tag_tid=4, entry_tid=3' = 'object_number=3, tag_tid=4, entry_tid=3' 2026-02-22T21:59:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent test_parent 2026-02-22T21:59:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying primary_position 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:23.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror_parent test_parent up+replaying primary_position '' 2026-02-22T21:59:24.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:24.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/test_parent 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_parent: 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: aa246585-a2e6-4705-9011-effcb6e76c43 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":4}} 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:22 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:24.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:21' 2026-02-22T21:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:59:24.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":4}} 2026-02-22T21:59:24.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:24.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:24.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:24.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:22' 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent test_parent up+replaying '2026-02-22 21:59:22' 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:22' 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:24.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format xml 2026-02-22T21:59:24.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143184337mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244343mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304347mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394349mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_parentaa246585-a2e6-4705-9011-effcb6e76c43up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":1,"object_number":1,"tag_tid":3},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":4}}43184337mirror.0trial1042026-02-22 21:59:22cluster2fae37073-621f-41d4-8a2a-f51b5c40d29cup+stoppedlocal image is primary2026-02-22 21:59:21' 2026-02-22T21:59:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:24.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/last_update' 2026-02-22T21:59:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:22' 2026-02-22T21:59:24.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_parent'\'']/state' 2026-02-22T21:59:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:59:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:59:24.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror_parent test_parent 2026-02-22T21:59:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:24.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:59:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror_parent-test_parent.export 2026-02-22T21:59:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror_parent-test_parent.export 2026-02-22T21:59:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.export 2026-02-22T21:59:24.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror_parent-test_parent.export 2026-02-22T21:59:24.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror_parent-test_parent.export 2026-02-22T21:59:24.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror_parent-test_parent.export 2026-02-22T21:59:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_parent-test_parent.export 2026-02-22T21:59:24.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror_parent/test_parent /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.export 2026-02-22T21:59:24.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:24.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror_parent/test_parent /tmp/tmp.S4fo03ccUK/cluster1-mirror_parent-test_parent.export 2026-02-22T21:59:24.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:24.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_parent-test_parent.export 2026-02-22T21:59:25.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_parent-test_parent.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_parent-test_parent.export 2026-02-22T21:59:25.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone started 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:59:25.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:25.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:26.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_clone started 2026-02-22T21:59:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:59:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:26.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:26.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:26.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone 2026-02-22T21:59:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:26.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:26.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:26.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:26.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:26.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone 2026-02-22T21:59:26.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:26.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:26.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_clone 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:59:26.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test_clone 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T21:59:26.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T21:59:26.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_clone 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_clone ']' 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_clone' 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_clone 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:26.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T21:59:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T21:59:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:26.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_clone 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T21:59:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:26.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:26.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_clone 2026-02-22T21:59:26.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_clone 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_clone '' 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_clone.status 2026-02-22T21:59:26.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_clone.status 2026-02-22T21:59:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.status 2026-02-22T21:59:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_clone 2026-02-22T21:59:26.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.status 2026-02-22T21:59:26.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:26.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:26.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:26.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:26.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:26.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.status 2026-02-22T21:59:26.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:59:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_clone 2026-02-22T21:59:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone 2026-02-22T21:59:26.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_clone '..*' 2026-02-22T21:59:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone 2026-02-22T21:59:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T21:59:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_clone.status 2026-02-22T21:59:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_clone.status 2026-02-22T21:59:26.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.status 2026-02-22T21:59:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_clone 2026-02-22T21:59:26.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.status 2026-02-22T21:59:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T21:59:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T21:59:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T21:59:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:26.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T21:59:26.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.status 2026-02-22T21:59:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:59:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T21:59:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:26.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_clone 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_clone up+replaying primary_position 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:59:26.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:26.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_clone up+replaying primary_position '' 2026-02-22T21:59:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:27.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T21:59:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T21:59:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:27.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_clone 2026-02-22T21:59:27.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_clone: 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 569cd20b-4030-479a-8ad6-51efd4f3c8e9 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:23 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T21:59:27.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 21:59:26' 2026-02-22T21:59:27.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T21:59:27.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T21:59:27.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T21:59:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T21:59:27.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T21:59:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T21:59:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:27.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T21:59:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T21:59:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T21:59:27.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:23' 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_clone up+replaying '2026-02-22 21:59:23' 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 21:59:23' 2026-02-22T21:59:27.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T21:59:27.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T21:59:27.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T21:59:27.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":2.4,"entries_behind_primary":0,"entries_per_second":0.1,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}43184322mirror.0trial1042026-02-22 21:59:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:59:26test_clone569cd20b-4030-479a-8ad6-51efd4f3c8e9up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 21:59:23cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 21:59:26' 2026-02-22T21:59:27.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T21:59:27.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/last_update' 2026-02-22T21:59:27.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 21:59:23' 2026-02-22T21:59:27.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_clone'\'']/state' 2026-02-22T21:59:27.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T21:59:27.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T21:59:27.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T21:59:27.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test_clone 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test_clone.export 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test_clone.export 2026-02-22T21:59:27.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.export 2026-02-22T21:59:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test_clone.export 2026-02-22T21:59:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test_clone.export 2026-02-22T21:59:27.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T21:59:27.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test_clone.export 2026-02-22T21:59:27.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_clone.export 2026-02-22T21:59:27.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test_clone /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.export 2026-02-22T21:59:27.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:27.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test_clone /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_clone.export 2026-02-22T21:59:27.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T21:59:27.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_clone.export 2026-02-22T21:59:27.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_clone.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_clone.export 2026-02-22T21:59:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test_clone 2026-02-22T21:59:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:59:27.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_clone 2026-02-22T21:59:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:27.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone 2026-02-22T21:59:27.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone 2026-02-22T21:59:28.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone 2026-02-22T21:59:28.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:59:28.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:28.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - clone v1' 2026-02-22T21:59:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:59:28.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:59:28 - clone v1 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:59:28 - clone v1 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap mirror test_clone1 journal 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T21:59:28.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone1 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image cluster1 mirror_parent test_parent snap mirror test_clone1 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone1 2026-02-22T21:59:28.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T21:59:28.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap mirror/test_clone1 --image-feature layering,exclusive-lock,journaling 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap mirror test_clone_v1 journal --rbd-default-clone-format 1 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap 2026-02-22T21:59:28.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone_v1 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image cluster2 mirror_parent test_parent snap mirror test_clone_v1 --rbd-default-clone-format 1 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap 2026-02-22T21:59:28.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T21:59:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone_v1 2026-02-22T21:59:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T21:59:28.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap mirror/test_clone_v1 --image-feature layering,exclusive-lock,journaling --rbd-default-clone-format 1 2026-02-22T21:59:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:59:28.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_clone_format cluster2 mirror test_clone_v1 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone_v1 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v1 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk 'BEGIN { 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 1 2026-02-22T21:59:28.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: $1 == "parent:" { 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: parent = $2 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: /op_features: .*clone-child/ { 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 2 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: END { 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: if (!parent) exit 1 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: print format 2026-02-22T21:59:28.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: }' 2026-02-22T21:59:28.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 1 = 1 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v1 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v1 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v1 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:59:28.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:28.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_clone_v1 started 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v1 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:29.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:29.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:29.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:29.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:29.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:29.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:29.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:29.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:29.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:29.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:29.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:29.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:29.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:29.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:29.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:29.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:29.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:29.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:29.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:29.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:29.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:29.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:29.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:29.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:29.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:29.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:29.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:59:30.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:30.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:30.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:30.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:30.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:30.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:30.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:30.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:30.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:30.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:30.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:30.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:30.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:30.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:30.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:30.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:30.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:30.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:30.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:30.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:30.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:30.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v1'] 2026-02-22T21:59:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:30.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:30.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:30.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T21:59:32.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:32.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:32.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:32.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:32.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:32.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:32.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:32.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v1 2026-02-22T21:59:32.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:59:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:59:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:59:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_clone_format cluster1 mirror test_clone_v1 2026-02-22T21:59:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone_v1 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v1 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk 'BEGIN { 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 1 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: $1 == "parent:" { 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: parent = $2 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: /op_features: .*clone-child/ { 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 2 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:32.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: END { 2026-02-22T21:59:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: if (!parent) exit 1 2026-02-22T21:59:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: print format 2026-02-22T21:59:32.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: }' 2026-02-22T21:59:32.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 1 = 1 2026-02-22T21:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test_clone_v1 2026-02-22T21:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v1 2026-02-22T21:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:59:32.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_clone_v1 2026-02-22T21:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v1 2026-02-22T21:59:32.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v1 2026-02-22T21:59:32.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v1 2026-02-22T21:59:32.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:59:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster1 mirror test_clone1 2026-02-22T21:59:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone1 2026-02-22T21:59:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:59:32.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster1 mirror test_clone1 2026-02-22T21:59:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:32.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone1 2026-02-22T21:59:32.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone1 2026-02-22T21:59:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone1 2026-02-22T21:59:32.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ unprotect_snapshot_retry cluster2 mirror_parent test_parent snap 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:59:32.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ unprotect_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T21:59:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:32.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap 2026-02-22T21:59:32.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap unprotect mirror_parent/test_parent@snap 2026-02-22T21:59:32.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap 2026-02-22T21:59:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap 2026-02-22T21:59:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap 2026-02-22T21:59:33.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing snap: 100% complete...done. 2026-02-22T21:59:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - clone v2' 2026-02-22T21:59:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:59:33.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:59:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:59:33 - clone v2 2026-02-22T21:59:33.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:59:33 - clone v2 2026-02-22T21:59:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ parent_snap=snap_v2 2026-02-22T21:59:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T21:59:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_v2 2026-02-22T21:59:33.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-22T21:59:34.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T21:59:34.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T21:59:34.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image_and_enable_mirror cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 journal --rbd-default-clone-format 2 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap_v2 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone_v2 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image cluster2 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-22T21:59:34.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T21:59:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T21:59:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap_v2 2026-02-22T21:59:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T21:59:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone_v2 2026-02-22T21:59:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T21:59:34.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock,journaling --rbd-default-clone-format 2 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_clone_format cluster2 mirror test_clone_v2 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone_v2 2026-02-22T21:59:34.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 info mirror/test_clone_v2 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk 'BEGIN { 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 1 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: $1 == "parent:" { 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: parent = $2 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: /op_features: .*clone-child/ { 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 2 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: END { 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: if (!parent) exit 1 2026-02-22T21:59:34.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: print format 2026-02-22T21:59:34.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: }' 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 2 = 2 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_clone_v2 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T21:59:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T21:59:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:34.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_clone_v2 started 2026-02-22T21:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T21:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T21:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T21:59:35.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:35.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T21:59:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:35.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:35.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:35.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:35.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:35.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:35.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:35.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:35.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:35.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:35.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:35.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:35.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:35.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:35.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:35.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:35.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:35.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:35.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:35.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:35.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:35.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:35.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:35.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:35.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:35.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:35.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:35.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T21:59:36.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:36.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:36.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:36.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:36.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:36.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:36.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:36.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:36.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:36.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:36.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:36.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:36.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:36.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:36.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:36.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:36.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:36.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:36.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:36.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:36.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:36.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:36.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:36.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:36.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:36.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:36.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:36.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:36.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:36.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:36.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:36.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:36.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T21:59:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:38.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:38.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:38.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:38.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:38.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:38.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:38.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:38.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:38.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:38.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:38.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:38.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:38.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:38.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:38.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:38.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:38.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:38.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:38.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:38.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:38.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:38.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:39.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:39.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:39.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:39.004 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:39.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:39.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:39.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:39.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 4 2026-02-22T21:59:43.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:43.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:43.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:43.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:43.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:43.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:43.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:43.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:43.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T21:59:43.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:43.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T21:59:43.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T21:59:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:43.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T21:59:43.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:43.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:43.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T21:59:43.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:43.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T21:59:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T21:59:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T21:59:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T21:59:43.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T21:59:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T21:59:43.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:43.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T21:59:43.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T21:59:43.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T21:59:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T21:59:43.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_clone_v2'] 2026-02-22T21:59:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T21:59:43.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T21:59:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T21:59:43.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 8 2026-02-22T21:59:51.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T21:59:51.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T21:59:51.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T21:59:51.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T21:59:51.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T21:59:51.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T21:59:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T21:59:51.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_clone_v2 2026-02-22T21:59:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T21:59:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T21:59:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T21:59:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T21:59:51.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_clone_format cluster1 mirror test_clone_v2 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone_v2 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 info mirror/test_clone_v2 2026-02-22T21:59:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk 'BEGIN { 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 1 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: $1 == "parent:" { 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: parent = $2 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: /op_features: .*clone-child/ { 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: format = 2 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: } 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: END { 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: if (!parent) exit 1 2026-02-22T21:59:51.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: print format 2026-02-22T21:59:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: }' 2026-02-22T21:59:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 2 = 2 2026-02-22T21:59:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T21:59:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_v2 2026-02-22T21:59:51.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-22T21:59:51.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing snap: 100% complete...done. 2026-02-22T21:59:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T21:59:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T21:59:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T21:59:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T21:59:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T21:59:51.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 5 e99592e0-0460-4dc2-a086-894eb2eb8306 128 MiB Sun Feb 22 21:59:34 2026 trash (user snap_v2) 2026-02-22T21:59:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T21:59:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:51.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T21:59:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T21:59:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T21:59:52.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T21:59:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T21:59:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T21:59:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T21:59:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T21:59:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T21:59:54.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T21:59:54.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 5 86657c2e-349e-4f86-b3eb-38e1495ae630 128 MiB Sun Feb 22 21:59:37 2026 trash (user snap_v2) 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test_clone_v2 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_clone_v2 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:54.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T21:59:54.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_clone_v2 2026-02-22T21:59:54.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_clone_v2 2026-02-22T21:59:55.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T21:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test_clone_v2 deleted 2026-02-22T21:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T21:59:55.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone_v2 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T21:59:55.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:59:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-22T21:59:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T21:59:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T21:59:55.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T21:59:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test_clone_v2 deleted '' 2026-02-22T21:59:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T21:59:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T21:59:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T21:59:55.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test_clone_v2 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_clone_v2 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test_clone_v2 2026-02-22T21:59:55.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T21:59:55.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test_clone_v2: (2) No such file or directory 2026-02-22T21:59:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T21:59:55.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T21:59:55.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T21:59:55.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:55.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T21:59:55.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:55.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:55.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T21:59:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T21:59:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:56.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T21:59:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T21:59:56.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T21:59:56.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:56.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T21:59:56.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - clone v2 non-primary' 2026-02-22T21:59:56.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T21:59:56.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T21:59:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 21:59:56 - clone v2 non-primary 2026-02-22T21:59:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 21:59:56 - clone v2 non-primary 2026-02-22T21:59:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T21:59:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T21:59:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_v2 2026-02-22T21:59:56.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror_parent/test_parent@snap_v2 2026-02-22T21:59:57.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T21:59:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T21:59:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror_parent test_parent snap_v2 2026-02-22T21:59:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T21:59:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T21:59:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T21:59:57.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T21:59:57.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T21:59:57.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:57.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T21:59:58.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror_parent/test_parent@snap_v2 2026-02-22T21:59:58.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T21:59:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T21:59:58.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T21:59:58.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:00.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror_parent/test_parent@snap_v2 2026-02-22T22:00:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test_parent': 2026-02-22T22:00:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:00.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 1 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10f1377991b7 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10f1377991b7 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 21:59:19 2026 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 21:59:19 2026 2026-02-22T22:00:00.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 21:59:19 2026 2026-02-22T22:00:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: False 2026-02-22T22:00:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10f1377991b7 2026-02-22T22:00:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:00:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:00:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: aa246585-a2e6-4705-9011-effcb6e76c43 2026-02-22T22:00:00.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:00:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image_and_enable_mirror cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 journal --rbd-default-clone-format 2 2026-02-22T22:00:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T22:00:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T22:00:00.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap_v2 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone_v2 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ clone_image cluster1 mirror_parent test_parent snap_v2 mirror test_clone_v2 --rbd-default-clone-format 2 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_pool=mirror_parent 2026-02-22T22:00:00.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_image=test_parent 2026-02-22T22:00:00.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local parent_snap=snap_v2 2026-02-22T22:00:00.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_pool=mirror 2026-02-22T22:00:00.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local clone_image=test_clone_v2 2026-02-22T22:00:00.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 6 2026-02-22T22:00:00.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 clone mirror_parent/test_parent@snap_v2 mirror/test_clone_v2 --image-feature layering,exclusive-lock,journaling --rbd-default-clone-format 2 2026-02-22T22:00:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:00:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:00:00.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:00:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:00:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_snapshot cluster2 mirror_parent test_parent snap_v2 2026-02-22T22:00:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T22:00:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T22:00:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_v2 2026-02-22T22:00:00.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rm mirror_parent/test_parent@snap_v2 2026-02-22T22:00:01.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing snap: 100% complete...done. 2026-02-22T22:00:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_removed_from_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T22:00:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster2 mirror_parent test_parent snap_v2 2026-02-22T22:00:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T22:00:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T22:00:01.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T22:00:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap ls mirror_parent/test_parent --all 2026-02-22T22:00:01.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T22:00:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:00:01.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T22:00:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:01.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T22:00:02.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T22:00:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T22:00:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T22:00:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T22:00:02.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T22:00:02.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 6 3704378b-cc4e-4998-8b2e-973915c2fc33 128 MiB Sun Feb 22 21:59:59 2026 trash (user snap_v2) 2026-02-22T22:00:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:02.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster1 mirror test_clone_v2 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster1 mirror test_clone_v2 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_clone_v2 2026-02-22T22:00:02.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_clone_v2 2026-02-22T22:00:02.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 rm mirror/test_clone_v2 2026-02-22T22:00:03.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:00:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T22:00:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:03.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:04.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_removed_from_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T22:00:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_snap_moved_to_trash cluster1 mirror_parent test_parent snap_v2 2026-02-22T22:00:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T22:00:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T22:00:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_v2 2026-02-22T22:00:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 snap ls mirror_parent/test_parent --all 2026-02-22T22:00:04.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -F ' trash (user snap_v2)' 2026-02-22T22:00:04.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror_parent test_parent 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:04.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror_parent test_parent 2026-02-22T22:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror_parent 2026-02-22T22:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_parent 2026-02-22T22:00:04.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror_parent/test_parent 2026-02-22T22:00:04.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror_parent/test_parent 2026-02-22T22:00:04.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:00:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: data pool' 2026-02-22T22:00:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:00:04.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:00:04 TEST: data pool 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:00:04 TEST: data pool 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ dp_image=test_data_pool 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test_data_pool journal 128 --data-pool mirror_parent 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test_data_pool 128 --data-pool mirror_parent 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:00:04.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 128 ']' 2026-02-22T22:00:04.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=128 2026-02-22T22:00:04.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:04.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling --data-pool mirror_parent mirror/test_data_pool 2026-02-22T22:00:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:00:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:00:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:00:04.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:00:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_data_pool cluster2 mirror test_data_pool 2026-02-22T22:00:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 info mirror/test_data_pool 2026-02-22T22:00:04.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T22:00:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ data_pool=mirror_parent 2026-02-22T22:00:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test mirror_parent = mirror_parent 2026-02-22T22:00:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_data_pool 2026-02-22T22:00:04.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_data_pool started 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_data_pool started 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:00:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:00:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:00:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:00:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:05.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:00:05.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:05.485 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:05.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:00:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:05.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:05.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:05.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:05.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:05.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:05.540 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:05.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:05.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:00:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:00:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:05.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:05.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:05.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:00:05.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:00:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:00:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:05.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:05.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:05.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:00:05.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:00:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:05.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:05.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:05.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:05.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:00:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:00:06.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:06.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:06.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:06.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:00:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:00:06.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:06.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:00:06.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:00:06.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:00:06.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:00:06.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:00:06.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:00:06.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:06.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:06.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:00:06.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:00:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:00:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:00:06.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:00:06.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:00:06.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:06.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:06.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:00:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:00:06.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:00:06.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:00:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:00:06.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:00:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:00:06.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:06.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_data_pool'] 2026-02-22T22:00:06.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:00:06.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:00:06.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:06.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T22:00:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:08.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:00:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_data_pool 2026-02-22T22:00:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:08.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:00:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:00:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:08.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_data_pool 2026-02-22T22:00:08.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:00:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:08.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:00:08.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_data_pool cluster1 mirror test_data_pool 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 info mirror/test_data_pool 2026-02-22T22:00:08.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ data_pool=mirror_parent 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test mirror_parent = mirror_parent 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test_data_pool snap1 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:09.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap1 2026-02-22T22:00:09.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-22T22:00:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:09.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:09.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:09.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:09.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:09.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:00:09.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test_data_pool snap2 2026-02-22T22:00:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:09.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test_data_pool@snap2 2026-02-22T22:00:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_data_pool 100 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:10.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:10.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 6250.13 bytes/sec: 24 MiB/s 2026-02-22T22:00:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_data_pool 2026-02-22T22:00:10.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test_data_pool 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:00:10.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:00:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:00:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:10.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:00:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_data_pool 2026-02-22T22:00:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:10.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_data_pool' 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:10.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:10.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:10.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:10.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:10.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:10.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:10.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '' 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:10.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:10.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:10.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '..*' 2026-02-22T22:00:10.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:10.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:10.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:10.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:00:10.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:10.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=5 2026-02-22T22:00:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:00:10.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=1 2026-02-22T22:00:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:00:10.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:10.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T22:00:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:00:10.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:10.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=3 2026-02-22T22:00:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 5 = 1 -a 101 -le 3 2026-02-22T22:00:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:00:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:10.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_data_pool 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_data_pool' 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:10.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:10.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:10.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:10.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:10.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:10.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:10.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:10.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:10.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '' 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:10.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:11.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '..*' 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:11.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:11.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:11.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T22:00:11.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_data_pool 2026-02-22T22:00:11.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:11.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:11.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:11.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_data_pool' 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:11.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:11.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:11.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:11.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:11.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:11.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:11.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:11.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:11.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:11.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '' 2026-02-22T22:00:11.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:11.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:11.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2]]], state=connected] 2026-02-22T22:00:11.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '..*' 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:11.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:11.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:11.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:11.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:11.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2]]], state=connected] 2026-02-22T22:00:11.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:00:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:11.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:00:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:11.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=5 2026-02-22T22:00:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:00:11.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=2 2026-02-22T22:00:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:11.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:00:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:11.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T22:00:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:00:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=1 2026-02-22T22:00:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 5 = 2 -a 101 -le 1 2026-02-22T22:00:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:00:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:00:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_data_pool 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_data_pool' 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:11.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:11.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:11.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '' 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:11.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:11.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2]]], state=connected] 2026-02-22T22:00:12.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:12.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '..*' 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:12.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2]]], state=connected] 2026-02-22T22:00:12.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:12.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:12.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T22:00:12.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_data_pool 2026-02-22T22:00:12.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:12.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:12.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:12.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_data_pool' 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:12.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:12.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:12.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:12.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:12.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:12.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:12.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:12.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:12.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:12.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:12.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:12.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:12.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:12.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '' 2026-02-22T22:00:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:12.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:12.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '..*' 2026-02-22T22:00:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:00:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:00:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:00:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:12.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=5 2026-02-22T22:00:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:00:12.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=3 2026-02-22T22:00:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:00:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T22:00:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:00:12.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:00:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=101 2026-02-22T22:00:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 5 = 3 -a 101 -le 101 2026-02-22T22:00:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:00:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:12.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:00:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_data_pool 2026-02-22T22:00:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:12.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_data_pool' 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:12.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:12.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:12.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:12.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:12.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '' 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:12.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:12.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:12.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '..*' 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:12.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:12.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:12.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=3, entry_tid=101], [object_number=0, tag_tid=3, entry_tid=100], [object_number=3, tag_tid=3, entry_tid=99], [object_number=2, tag_tid=3, entry_tid=98]]], state=connected] 2026-02-22T22:00:12.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:12.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=3, entry_tid=101' '!=' 'object_number=1, tag_tid=3, entry_tid=101' 2026-02-22T22:00:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:12.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T22:00:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_data_pool 2026-02-22T22:00:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:13.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_data_pool' 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:13.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:13.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:13.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:13.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:13.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:13.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:13.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:13.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_data_pool 2026-02-22T22:00:13.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:13.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '' 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:13.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:13.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:13.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:13.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:13.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_data_pool 2026-02-22T22:00:13.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:13.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:13.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:13.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_data_pool '..*' 2026-02-22T22:00:13.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:00:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_data_pool 2026-02-22T22:00:13.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=5, entry_tid=101], [object_number=0, tag_tid=5, entry_tid=100], [object_number=3, tag_tid=5, entry_tid=99], [object_number=2, tag_tid=5, entry_tid=98]]], state=connected] 2026-02-22T22:00:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.status 2026-02-22T22:00:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=5, entry_tid=101' -a 'object_number=1, tag_tid=5, entry_tid=101' = 'object_number=1, tag_tid=5, entry_tid=101' 2026-02-22T22:00:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_data_pool 2026-02-22T22:00:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:13.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_data_pool up+replaying primary_position 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:13.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_data_pool up+replaying primary_position '' 2026-02-22T22:00:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:00:14.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:00:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:00:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:00:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:00:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_data_pool 2026-02-22T22:00:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_data_pool: 2026-02-22T22:00:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 9b56dc4e-3108-437a-93ee-0ea93ba5f9d2 2026-02-22T22:00:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:00:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:00:14.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:00:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:00:07 2026-02-22T22:00:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:00:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:00:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:00:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:00:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:00:11' 2026-02-22T22:00:14.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:00:14.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:00:14.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:00:14.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:00:14.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:00:14.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:00:14.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:00:14.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:00:14.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:00:14.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:00:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:00:14.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:00:14.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:00:07' 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_data_pool up+replaying '2026-02-22 22:00:07' 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:00:07' 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:00:14.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:00:14.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}43184322mirror.0trial1042026-02-22 21:59:46cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:00:11test_data_pool9b56dc4e-3108-437a-93ee-0ea93ba5f9d2up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43184322mirror.0trial1042026-02-22 22:00:07cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:00:11' 2026-02-22T22:00:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:00:14.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-22T22:00:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:00:07' 2026-02-22T22:00:14.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-22T22:00:14.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:00:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:00:14.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:00:14.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test_data_pool@snap1 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool@snap1 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap1 /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:14.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap1 /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:14.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap1.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap1.export 2026-02-22T22:00:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test_data_pool@snap2 2026-02-22T22:00:14.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool@snap2 2026-02-22T22:00:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:00:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool@snap2 /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:14.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool@snap2 /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T22:00:14.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:14.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T22:00:15.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool@snap2.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool@snap2.export 2026-02-22T22:00:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test_data_pool 2026-02-22T22:00:15.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:00:15.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test_data_pool.export 2026-02-22T22:00:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test_data_pool.export 2026-02-22T22:00:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.export 2026-02-22T22:00:15.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test_data_pool.export 2026-02-22T22:00:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test_data_pool.export 2026-02-22T22:00:15.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool.export 2026-02-22T22:00:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool.export 2026-02-22T22:00:15.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test_data_pool /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.export 2026-02-22T22:00:15.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:15.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test_data_pool /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool.export 2026-02-22T22:00:15.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:15.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool.export 2026-02-22T22:00:15.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_data_pool.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_data_pool.export 2026-02-22T22:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:15.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test_data_pool 2026-02-22T22:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:00:15.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_data_pool 2026-02-22T22:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:00:15.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_data_pool 2026-02-22T22:00:17.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T22:00:17.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_data_pool 2026-02-22T22:00:17.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:00:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: disable mirroring / delete non-primary image' 2026-02-22T22:00:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:00:17.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:00:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:00:17 TEST: disable mirroring / delete non-primary image 2026-02-22T22:00:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:00:17 TEST: disable mirroring / delete non-primary image 2026-02-22T22:00:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image2=test2 2026-02-22T22:00:17.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image3=test3 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image4=test4 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image5=test5 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test2 journal 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:17.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test2 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:00:17.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:00:17.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test2 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:17.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:17.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:17.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:17.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:17.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:00:17.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test2 snap1 2026-02-22T22:00:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:17.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap1 2026-02-22T22:00:18.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:18.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test2 snap2 2026-02-22T22:00:18.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:18.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:18.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:18.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:18.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap2 2026-02-22T22:00:19.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' test2 = test4 ']' 2026-02-22T22:00:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' test2 = test5 ']' 2026-02-22T22:00:19.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test2 100 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:19.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:19.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:19.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test2 present 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:19.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:19.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id=10e2595ae101 2026-02-22T22:00:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:19.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 present 10e2595ae101 2026-02-22T22:00:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:19.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e2595ae101 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:19.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:19.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e2595ae101 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e2595ae101 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 10e2595ae101 -o 10e2595ae101 = 10e2595ae101 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test2 snap2 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap2 2026-02-22T22:00:19.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:19.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:19.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:20.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T22:00:20.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:00:20.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:00:20.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:20.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:22.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test2': 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 2 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10e2595ae101 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10e2595ae101 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:00:22.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:00:19 2026 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:00:19 2026 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:00:19 2026 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: False 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10e2595ae101 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: 34772ec9-f1d0-4028-8847-9256f0c37127 2026-02-22T22:00:22.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:00:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test3 journal 2026-02-22T22:00:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:22.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test3 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:22.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:22.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:22.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:22.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:00:22.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:00:22.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test3 2026-02-22T22:00:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:00:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:00:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:00:22.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:22.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:22.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:22.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T22:00:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test3 snap1 2026-02-22T22:00:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:22.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:22.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap1 2026-02-22T22:00:23.283 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T22:00:23.282+0000 7f46b35cb640 -1 reset not still connected to 0x5651b1e72c30 2026-02-22T22:00:23.283 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T22:00:23.283+0000 7f46b35cb640 -1 reset not still connected to 0x5651b26b3110 2026-02-22T22:00:23.284 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T22:00:23.283+0000 7f46b35cb640 -1 reset not still connected to 0x5651b27fed00 2026-02-22T22:00:23.284 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T22:00:23.283+0000 7f46b35cb640 -1 reset not still connected to 0x5651b3c6fee0 2026-02-22T22:00:23.284 INFO:tasks.ceph.cluster2.osd.2.trial104.stderr:2026-02-22T22:00:23.283+0000 7f46b35cb640 -1 reset not still connected to 0x5651b3c75860 2026-02-22T22:00:23.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test3 snap2 2026-02-22T22:00:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:23.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:23.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test3@snap2 2026-02-22T22:00:24.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' test3 = test4 ']' 2026-02-22T22:00:24.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' test3 = test5 ']' 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test3 100 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:24.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test3 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:24.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:24.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:24.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7143 bytes/sec: 28 MiB/s 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test3 present 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:24.360 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T22:00:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test3 2026-02-22T22:00:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T22:00:24.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:24.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T22:00:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:00:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:24.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test3 present '' 2026-02-22T22:00:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:24.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T22:00:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test3 2026-02-22T22:00:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T22:00:24.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:24.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T22:00:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:00:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:24.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:25.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test3 present '' 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:25.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test3 2026-02-22T22:00:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T22:00:25.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:25.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e24d7b9517 2026-02-22T22:00:25.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e24d7b9517 2026-02-22T22:00:25.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 10e24d7b9517 2026-02-22T22:00:25.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test3 snap2 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap2 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:25.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:26.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-22T22:00:26.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:00:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:00:26.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:26.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:28.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test3@snap2 2026-02-22T22:00:28.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test3': 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 3 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10e24d7b9517 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10e24d7b9517 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:00:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:00:24 2026 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:00:24 2026 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:00:24 2026 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: False 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10e24d7b9517 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: d3cf5960-6e62-4763-8ffb-9aa8c5a197bc 2026-02-22T22:00:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:00:28.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:28.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:28.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test4 journal 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test4 2026-02-22T22:00:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:00:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test4 2026-02-22T22:00:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:00:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:00:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:00:28.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:28.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:28.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:28.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:00:28.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test4 snap1 2026-02-22T22:00:28.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:28.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:28.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:28.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:28.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap1 2026-02-22T22:00:29.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test4 snap2 2026-02-22T22:00:29.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:29.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test4@snap2 2026-02-22T22:00:30.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' test4 = test4 ']' 2026-02-22T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ protect_snapshot cluster2 mirror test4 snap1 2026-02-22T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap1 2026-02-22T22:00:30.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ protect_snapshot cluster2 mirror test4 snap2 2026-02-22T22:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:30.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap protect mirror/test4@snap2 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test4 100 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:30.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test4 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:30.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:30.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:00:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:00:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test4 present 2026-02-22T22:00:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:00:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:00:30.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:30.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T22:00:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:00:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:30.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:30.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test4 present '' 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:30.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:00:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:00:30.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:30.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T22:00:30.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:30.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:00:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:30.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:31.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test4 present '' 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:00:31.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:00:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:00:31.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e2cb3f1263 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e2cb3f1263 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 10e2cb3f1263 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test4 snap2 2026-02-22T22:00:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap2 2026-02-22T22:00:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:32.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-02-22T22:00:32.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:00:32.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:00:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:32.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:34.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-02-22T22:00:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test4': 2026-02-22T22:00:34.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 3 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10e2cb3f1263 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10e2cb3f1263 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling, operations 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: snap-trash 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:00:30 2026 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:00:30 2026 2026-02-22T22:00:34.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:00:30 2026 2026-02-22T22:00:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: True 2026-02-22T22:00:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10e2cb3f1263 2026-02-22T22:00:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:00:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:00:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: b4452b31-e55f-4fe1-a70d-a8f9ae13b361 2026-02-22T22:00:34.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:00:34.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:34.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:34.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test5 journal 2026-02-22T22:00:34.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test5 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:34.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:34.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:34.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:34.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:34.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:00:34.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:00:34.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test5 2026-02-22T22:00:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test5 100 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:34.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:34.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:34.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:34.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 9091.09 bytes/sec: 36 MiB/s 2026-02-22T22:00:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test5 snap1 2026-02-22T22:00:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:34.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap1 2026-02-22T22:00:35.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:35.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test5 snap2 2026-02-22T22:00:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:35.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test5@snap2 2026-02-22T22:00:36.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:00:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' test5 = test4 ']' 2026-02-22T22:00:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' test5 = test5 ']' 2026-02-22T22:00:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ protect_snapshot cluster2 mirror test5 snap1 2026-02-22T22:00:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:36.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:36.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:36.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap1 2026-02-22T22:00:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ protect_snapshot cluster2 mirror test5 snap2 2026-02-22T22:00:36.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:36.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap protect mirror/test5@snap2 2026-02-22T22:00:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test5 100 2026-02-22T22:00:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:00:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:00:36.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:00:36.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test5 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:00:36.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:00:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:00:36.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 6666.8 bytes/sec: 26 MiB/s 2026-02-22T22:00:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:00:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test5 present 2026-02-22T22:00:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:00:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test5 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T22:00:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:36.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T22:00:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:00:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:36.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test5 present '' 2026-02-22T22:00:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:36.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:36.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:36.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T22:00:36.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:36.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:36.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test5 2026-02-22T22:00:36.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T22:00:36.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:36.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T22:00:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:36.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:00:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:36.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test5 present '' 2026-02-22T22:00:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:37.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test5 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T22:00:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:37.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e22b2aca51 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e22b2aca51 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 10e22b2aca51 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test5 snap2 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap2 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:37.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:37.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:38.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-02-22T22:00:38.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:00:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:00:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:40.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test5@snap2 2026-02-22T22:00:40.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test5': 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 3 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10e22b2aca51 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10e22b2aca51 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling, operations 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: snap-trash 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:00:40.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:00:36 2026 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:00:36 2026 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:00:36 2026 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: True 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10e22b2aca51 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: 1c4d9b2a-8f65-46f6-a25c-d3880e082136 2026-02-22T22:00:40.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:00:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-22T22:00:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:40.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=image 2026-02-22T22:00:40.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-22T22:00:40.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:note: changing mirroring mode from pool to image 2026-02-22T22:00:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image4} 2026-02-22T22:00:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ disable_mirror cluster2 mirror test2 2026-02-22T22:00:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:40.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test2 2026-02-22T22:00:40.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Mirroring disabled 2026-02-22T22:00:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image4} 2026-02-22T22:00:40.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ disable_mirror cluster2 mirror test4 2026-02-22T22:00:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:40.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test4 2026-02-22T22:00:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Mirroring disabled 2026-02-22T22:00:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ unprotect_snapshot cluster2 mirror test5 snap1 2026-02-22T22:00:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:40.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:40.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap1 2026-02-22T22:00:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ unprotect_snapshot cluster2 mirror test5 snap2 2026-02-22T22:00:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:40.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:40.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test5@snap2 2026-02-22T22:00:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image3} ${image5} 2026-02-22T22:00:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_snapshot cluster2 mirror test3 snap1 2026-02-22T22:00:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:40.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap1 2026-02-22T22:00:41.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing snap: 100% complete...done. 2026-02-22T22:00:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_snapshot cluster2 mirror test3 snap2 2026-02-22T22:00:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:41.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rm mirror/test3@snap2 2026-02-22T22:00:42.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing snap: 100% complete...done. 2026-02-22T22:00:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test3 2026-02-22T22:00:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:00:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test3 2026-02-22T22:00:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test3 2026-02-22T22:00:42.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test3 2026-02-22T22:00:42.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:00:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:42.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image3} ${image5} 2026-02-22T22:00:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_snapshot cluster2 mirror test5 snap1 2026-02-22T22:00:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:00:42.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap1 2026-02-22T22:00:43.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing snap: 100% complete...done. 2026-02-22T22:00:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_snapshot cluster2 mirror test5 snap2 2026-02-22T22:00:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:00:43.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rm mirror/test5@snap2 2026-02-22T22:00:44.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing snap: 100% complete...done. 2026-02-22T22:00:44.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test5 2026-02-22T22:00:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:00:44.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test5 2026-02-22T22:00:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:44.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test5 2026-02-22T22:00:44.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test5 2026-02-22T22:00:44.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:00:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test2 deleted 2026-02-22T22:00:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:44.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id=10e2595ae101 2026-02-22T22:00:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:44.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 deleted 10e2595ae101 2026-02-22T22:00:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:00:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e2595ae101 2026-02-22T22:00:44.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:44.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:44.537 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e2595ae101 2026-02-22T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e2595ae101 2026-02-22T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 10e2595ae101 -o 10e2595ae101 = 10e2595ae101 2026-02-22T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:00:44.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:44.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:45.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 deleted 10e2595ae101 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e2595ae101 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:45.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:45.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:45.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e2595ae101 2026-02-22T22:00:45.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e2595ae101 2026-02-22T22:00:45.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 10e2595ae101 -o 10e2595ae101 = 10e2595ae101 2026-02-22T22:00:45.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:00:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:00:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:45.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 deleted 10e2595ae101 2026-02-22T22:00:47.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:00:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e2595ae101 2026-02-22T22:00:47.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:47.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:47.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T22:00:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:00:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test3 deleted 2026-02-22T22:00:47.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test3 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T22:00:47.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:47.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T22:00:47.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:00:47.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:47.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:47.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test3 deleted '' 2026-02-22T22:00:47.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:47.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test3 2026-02-22T22:00:47.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:00:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test3 2026-02-22T22:00:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:47.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test3 2026-02-22T22:00:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test3 2026-02-22T22:00:47.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:47.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test3: (2) No such file or directory 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test4 deleted 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:00:47.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:47.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T22:00:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:00:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:47.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test4 deleted '' 2026-02-22T22:00:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:00:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:00:47.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:00:47.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:47.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T22:00:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:00:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:47.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image3} ${image4} ${image5} 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test5 deleted 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:47.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test5 2026-02-22T22:00:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T22:00:47.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:47.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T22:00:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:00:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:47.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test5 deleted '' 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test5 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:48.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test5 2026-02-22T22:00:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test5 2026-02-22T22:00:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test5 2026-02-22T22:00:48.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:48.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test5: (2) No such file or directory 2026-02-22T22:00:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:00:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:48.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:00:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_pool_mirror_mode cluster2 mirror pool 2026-02-22T22:00:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=pool 2026-02-22T22:00:48.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror pool 2026-02-22T22:00:48.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:note: changing mirroring mode from image to pool 2026-02-22T22:00:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image4} 2026-02-22T22:00:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ enable_journaling cluster2 mirror test2 2026-02-22T22:00:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:48.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 feature enable mirror/test2 journaling 2026-02-22T22:00:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test2 present 2026-02-22T22:00:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:48.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:48.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:48.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T22:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:00:48.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 present '' 2026-02-22T22:00:48.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:48.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:48.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:48.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:48.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T22:00:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:00:48.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:48.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 present '' 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:49.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:49.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T22:00:49.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:00:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:49.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:51.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 present '' 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:51.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:51.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:51.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test2: (2) No such file or directory 2026-02-22T22:00:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:00:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:00:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:00:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:00:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:00:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test2 present '' 2026-02-22T22:00:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:00:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test2 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test2 2026-02-22T22:00:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:00:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e2379d1612 2026-02-22T22:00:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e2379d1612 2026-02-22T22:00:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 10e2379d1612 2026-02-22T22:00:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:00:55.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test2 snap2 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap2 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:55.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:56.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T22:00:56.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:00:56.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:00:56.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:00:56.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:00:58.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap2 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test2': 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 3 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10e2379d1612 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10e2379d1612 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling, operations 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: snap-trash 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:00:55 2026 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:00:55 2026 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:00:55 2026 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: False 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10e2379d1612 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: 2e161c9c-920b-46da-b7ab-3890d84dfe0e 2026-02-22T22:00:58.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test2 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test2 started 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:00:58.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:58.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:00:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test2 started 2026-02-22T22:00:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:59.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:00:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:00:59.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:00:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test2 2026-02-22T22:00:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:00:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:00:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:00:59.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:00:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:00:59.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test2 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:59.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:59.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:00:59.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test2 2026-02-22T22:00:59.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:00:59.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:00:59.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:00:59.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:00:59.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test2 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test2 2026-02-22T22:00:59.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:00:59.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:00:59.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test2 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test2 ']' 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test2' 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test2 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:59.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:00:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:00:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:00:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:00:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:00:59.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:00:59.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test2 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:00:59.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:00:59.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:00:59.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:00:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:00:59.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test2 2026-02-22T22:00:59.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test2 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test2 '' 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:59.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:00:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:00:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:00:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.status 2026-02-22T22:00:59.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:00:59.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.status 2026-02-22T22:00:59.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:59.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:59.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:59.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:59.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:59.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.status 2026-02-22T22:00:59.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test2 2026-02-22T22:00:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:59.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test2 '..*' 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test2 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test2.status 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test2.status 2026-02-22T22:00:59.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.status 2026-02-22T22:00:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test2 2026-02-22T22:00:59.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.status 2026-02-22T22:00:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:00:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:00:59.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:00:59.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:59.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:00:59.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.status 2026-02-22T22:00:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:59.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=1, entry_tid=3' -a 'object_number=3, tag_tid=1, entry_tid=3' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test2 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test2.export 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test2.export 2026-02-22T22:00:59.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.export 2026-02-22T22:00:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test2.export 2026-02-22T22:00:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test2.export 2026-02-22T22:00:59.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:00:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test2.export 2026-02-22T22:00:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test2.export 2026-02-22T22:00:59.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test2 /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.export 2026-02-22T22:00:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:00:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test2 /tmp/tmp.S4fo03ccUK/cluster1-mirror-test2.export 2026-02-22T22:01:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:00.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test2.export 2026-02-22T22:01:00.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test2.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test2.export 2026-02-22T22:01:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image4} 2026-02-22T22:01:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ enable_journaling cluster2 mirror test4 2026-02-22T22:01:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:00.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:00.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 feature enable mirror/test4 journaling 2026-02-22T22:01:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test4 present 2026-02-22T22:01:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:00.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:01:00.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:01:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T22:01:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:01:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:01:00.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:01:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test4 present '' 2026-02-22T22:01:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:01:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:01:00.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:01:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:01:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:01:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:01:00.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:01:00.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T22:01:00.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:01:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:01:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:01:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:01:00.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:01.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test4 present '' 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:01:01.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:01:01.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:01:01.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test4: (2) No such file or directory 2026-02-22T22:01:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:01:01.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:01:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:01:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:01:01.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test4 present '' 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:01:03.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:01:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test4 2026-02-22T22:01:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test4 2026-02-22T22:01:03.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:01:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e27083cbcd 2026-02-22T22:01:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e27083cbcd 2026-02-22T22:01:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 10e27083cbcd 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test4 snap2 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap2 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:01:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:04.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test4@snap2 2026-02-22T22:01:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test4': 2026-02-22T22:01:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:01:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:01:04.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 2 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10e27083cbcd 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10e27083cbcd 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:01:02 2026 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:01:02 2026 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:01:02 2026 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: False 2026-02-22T22:01:04.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10e27083cbcd 2026-02-22T22:01:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:01:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:01:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: 8f84c2ef-1d33-4c3f-8bc9-942e1b4cacba 2026-02-22T22:01:04.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:01:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test4 2026-02-22T22:01:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:04.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test4 started 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:04.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test4 started 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:01:05.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test4 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:05.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:01:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:01:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:05.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test4 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:01:05.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:01:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:01:05.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:05.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:05.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:05.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:05.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test4 2026-02-22T22:01:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test4'] 2026-02-22T22:01:05.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:05.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test4 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:05.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:01:05.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:01:05.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:05.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test4 2026-02-22T22:01:05.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test4'] 2026-02-22T22:01:05.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:05.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:05.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test4 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:05.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:05.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:01:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:01:05.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:05.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test4 2026-02-22T22:01:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test4'] 2026-02-22T22:01:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:05.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test4 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:05.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:05.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:01:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:05.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:01:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:01:05.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test4 2026-02-22T22:01:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test4'] 2026-02-22T22:01:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:05.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:05.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:01:06.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:06.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:01:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:06.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test4 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:06.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:01:06.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:01:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:01:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:01:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:01:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:01:06.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:06.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:06.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test4 2026-02-22T22:01:06.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:01:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:01:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:01:06.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:01:06.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test4 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:06.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test4 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:06.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:01:06.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test4 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test4 ']' 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test4 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:01:06.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:06.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:06.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:01:06.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:06.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:06.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:06.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:06.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:06.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:06.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:06.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:06.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test4 2026-02-22T22:01:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:01:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:01:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:01:06.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:06.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:06.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:06.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:06.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:01:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:01:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:01:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:06.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:01:06.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:06.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:06.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=1, entry_tid=3' -a 'object_number=3, tag_tid=1, entry_tid=3' = 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=1' '!=' '' 2026-02-22T22:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=1' = '' 2026-02-22T22:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:01:06.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=1 2026-02-22T22:01:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:01:06.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:01:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=1 2026-02-22T22:01:06.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:01:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:01:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:01:06.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=3 2026-02-22T22:01:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:01:07.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:01:07.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=1 2026-02-22T22:01:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 1 = 1 -a 3 -le 1 2026-02-22T22:01:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:01:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:07.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test4 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test4 ']' 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test4 2026-02-22T22:01:07.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:07.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:07.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:07.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test4 2026-02-22T22:01:07.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:07.265 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:01:07.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:01:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:01:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:01:07.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:01:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:01:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=1, entry_tid=3' -a 'object_number=3, tag_tid=1, entry_tid=3' = 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=1, entry_tid=1' '!=' 'object_number=1, tag_tid=1, entry_tid=1' 2026-02-22T22:01:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:07.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T22:01:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test4 2026-02-22T22:01:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:07.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test4 ']' 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test4' 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test4 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:07.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test4 2026-02-22T22:01:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:07.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:07.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:07.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:07.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test4 2026-02-22T22:01:07.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test4 2026-02-22T22:01:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test4 '' 2026-02-22T22:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:01:07.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:01:07.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:07.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:01:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test4 2026-02-22T22:01:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test4 '..*' 2026-02-22T22:01:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test4 2026-02-22T22:01:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:01:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test4.status 2026-02-22T22:01:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test4.status 2026-02-22T22:01:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test4 2026-02-22T22:01:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:07.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:01:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.status 2026-02-22T22:01:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=1, entry_tid=3' -a 'object_number=3, tag_tid=1, entry_tid=3' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test4 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test4.export 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test4.export 2026-02-22T22:01:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.export 2026-02-22T22:01:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test4.export 2026-02-22T22:01:07.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test4.export 2026-02-22T22:01:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:07.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test4.export 2026-02-22T22:01:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test4.export 2026-02-22T22:01:07.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test4 /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.export 2026-02-22T22:01:07.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:07.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test4 /tmp/tmp.S4fo03ccUK/cluster1-mirror-test4.export 2026-02-22T22:01:08.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:01:08.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test4.export 2026-02-22T22:01:08.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test4.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test4.export 2026-02-22T22:01:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: remove mirroring pool' 2026-02-22T22:01:08.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:01:08.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:01:08 TEST: remove mirroring pool 2026-02-22T22:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:01:08 TEST: remove mirroring pool 2026-02-22T22:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pool=pool_to_remove 2026-02-22T22:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for cluster in ${CLUSTER1} ${CLUSTER2} 2026-02-22T22:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:01:08.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster1 osd pool create pool_to_remove 16 16 2026-02-22T22:01:08.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'pool_to_remove' already exists 2026-02-22T22:01:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:01:08.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 pool init pool_to_remove 2026-02-22T22:01:11.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 mirror pool enable pool_to_remove pool 2026-02-22T22:01:11.308 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for cluster in ${CLUSTER1} ${CLUSTER2} 2026-02-22T22:01:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:01:11.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster2 osd pool create pool_to_remove 16 16 2026-02-22T22:01:12.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'pool_to_remove' already exists 2026-02-22T22:01:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:01:12.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 pool init pool_to_remove 2026-02-22T22:01:15.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 mirror pool enable pool_to_remove pool 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_add cluster1 pool_to_remove cluster2 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local client_cluster=cluster2 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster2 2026-02-22T22:01:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid_var_name 2026-02-22T22:01:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:01:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local error_code 2026-02-22T22:01:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local peer_uuid 2026-02-22T22:01:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:01:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:01:15.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool peer add pool_to_remove cluster2 2026-02-22T22:01:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_uuid=f6c14d0e-0a05-4451-b5c5-b68c0326c29e 2026-02-22T22:01:15.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 -eq 0 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_add cluster2 pool_to_remove cluster1 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local client_cluster=cluster1 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local remote_cluster=cluster1 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local uuid_var_name 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local error_code 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local peer_uuid 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 16 32 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:01:15.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool peer add pool_to_remove cluster1 2026-02-22T22:01:15.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ peer_uuid=1735ca2e-47ed-431f-999b-b78c9b569053 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 17 ']' 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 -eq 0 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rdp_image=test_remove_data_pool 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 pool_to_remove test 128 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 128 ']' 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=128 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling pool_to_remove/test 2026-02-22T22:01:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test_remove_data_pool 128 --data-pool pool_to_remove 2026-02-22T22:01:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 128 ']' 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=128 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling --data-pool pool_to_remove mirror/test_remove_data_pool 2026-02-22T22:01:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 pool_to_remove test 100 2026-02-22T22:01:15.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:01:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:01:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:01:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:01:15.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench pool_to_remove/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:01:15.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:01:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:01:15.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:01:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_remove_data_pool 100 2026-02-22T22:01:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:01:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:01:15.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:01:15.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:01:15.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_remove_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:01:15.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:01:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:01:15.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:01:15.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 pool_to_remove test 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 pool_to_remove test 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:15.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:01:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:01:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:01:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:15.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:01:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 pool_to_remove test 2026-02-22T22:01:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:15.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush pool_to_remove/test' 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:15.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:15.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:15.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:15.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:15.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:01:15.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:15.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:15.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:15.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:15.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:15.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:15.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:15.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:15.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:15.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:15.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:15.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:15.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:15.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:15.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:01:15.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:15.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:15.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:15.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:15.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:15.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:15.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:16.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:16.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:16.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:16.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:16.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:16.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:16.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:16.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:16.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:16.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:17.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:17.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:17.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:17.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:17.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:17.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:17.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:17.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:17.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:17.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:17.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:17.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:17.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:17.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:17.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:17.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:17.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:01:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:17.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:01:17.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:17.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:17.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:17.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:17.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:17.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:17.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:17.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:01:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:19.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:19.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:19.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:19.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:19.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:19.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:19.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:19.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:19.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:19.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:19.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:19.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:2 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:01:19.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:19.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:01:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:01:19.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:19.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:3 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T22:01:19.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:19.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:01:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:01:19.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:19.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:19.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:01:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:23.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:23.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:23.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:23.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:23.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:01:23.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush pool_to_remove/test 2026-02-22T22:01:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:01:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:01:23.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:23.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:01:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:23.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:01:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:23.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:01:23.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush pool_to_remove/test 2026-02-22T22:01:23.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 pool_to_remove test 2026-02-22T22:01:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=pool_to_remove 2026-02-22T22:01:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:01:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 pool_to_remove test '' 2026-02-22T22:01:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:23.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=pool_to_remove 2026-02-22T22:01:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:01:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:01:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:23.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image pool_to_remove/test 2026-02-22T22:01:23.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:23.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:23.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=9d16f90a-9759-409b-8ac7-f98abda3153b, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:23.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 pool_to_remove test 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=pool_to_remove 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 pool_to_remove test '..*' 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=pool_to_remove 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:23.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image pool_to_remove/test 2026-02-22T22:01:23.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:23.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:23.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=9d16f90a-9759-409b-8ac7-f98abda3153b, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:23.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-pool_to_remove-test.status 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 pool_to_remove test 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 pool_to_remove test up+replaying primary_position 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:01:23.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:23.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 pool_to_remove test up+replaying primary_position '' 2026-02-22T22:01:24.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:01:24.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status pool_to_remove/test 2026-02-22T22:01:24.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T22:01:24.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 7b4dd0df-30c5-4609-b67c-98bb7c8fc400 2026-02-22T22:01:24.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:01:24.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:01:21 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:01:22' 2026-02-22T22:01:24.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:01:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:01:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:01:24.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:01:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:01:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:01:24.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:01:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:01:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:01:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:01:24.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:01:24.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:01:21' 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 pool_to_remove test up+replaying '2026-02-22 22:01:21' 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=pool_to_remove 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:01:21' 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:01:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:01:24.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status pool_to_remove --verbose --format xml 2026-02-22T22:01:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK143185671mirror.0trial10419.2.3-689-gf90f4eaefalseOK43245670mirror.1trial10419.2.3-689-gf90f4eaetrueOK43305674mirror.2trial10419.2.3-689-gf90f4eaefalseOK43395676mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest7b4dd0df-30c5-4609-b67c-98bb7c8fc400up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43245670mirror.1trial1042026-02-22 22:01:21cluster21d31c28e-4a44-4872-ab8a-b4460df03838up+stoppedlocal image is primary2026-02-22 22:01:22' 2026-02-22T22:01:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:01:24.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:01:24.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:01:21' 2026-02-22T22:01:24.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:01:24.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:01:24.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:01:24.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:01:24.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:01:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:24.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_remove_data_pool 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test_remove_data_pool 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:01:24.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:01:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:01:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:01:24.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:01:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_remove_data_pool 2026-02-22T22:01:25.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_remove_data_pool ']' 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_remove_data_pool' 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_remove_data_pool 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:01:25.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:01:25.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_remove_data_pool 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:01:25.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:01:25.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:25.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:01:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:01:25.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_remove_data_pool 2026-02-22T22:01:25.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_remove_data_pool 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_remove_data_pool 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_remove_data_pool '' 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:25.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_remove_data_pool 2026-02-22T22:01:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:01:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_remove_data_pool 2026-02-22T22:01:25.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:25.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:25.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:25.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_remove_data_pool 2026-02-22T22:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_remove_data_pool 2026-02-22T22:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_remove_data_pool '..*' 2026-02-22T22:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:25.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_remove_data_pool 2026-02-22T22:01:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:01:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:01:25.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_remove_data_pool 2026-02-22T22:01:25.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:01:25.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:01:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:01:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:25.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:01:25.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_remove_data_pool.status 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_remove_data_pool 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:01:25.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_remove_data_pool up+replaying primary_position 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_remove_data_pool up+replaying primary_position '' 2026-02-22T22:01:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:01:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:01:26.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:01:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:01:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:01:26.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_remove_data_pool 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_remove_data_pool: 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 7f3def0c-4f73-4b75-abc0-e815fba5e100 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:01:26.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:01:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:01:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:01:19 2026-02-22T22:01:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:01:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:01:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:01:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:01:26.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:01:21' 2026-02-22T22:01:26.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:01:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:01:26.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:01:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:01:26.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:01:26.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:01:26.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:01:26.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:01:26.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:01:26.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:01:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:01:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:01:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:01:19' 2026-02-22T22:01:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_remove_data_pool up+replaying '2026-02-22 22:01:19' 2026-02-22T22:01:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:26.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:01:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:01:19' 2026-02-22T22:01:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:01:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:01:26.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK443184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}43184322mirror.0trial1042026-02-22 22:01:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:01:21test22e161c9c-920b-46da-b7ab-3890d84dfe0eup+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43184322mirror.0trial1042026-02-22 22:01:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:01:21test48f84c2ef-1d33-4c3f-8bc9-942e1b4cacbaup+replayingreplaying, {"bytes_per_second":35.0,"entries_behind_primary":0,"entries_per_second":0.4,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}43184322mirror.0trial1042026-02-22 22:01:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:01:21test_remove_data_pool7f3def0c-4f73-4b75-abc0-e815fba5e100up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 22:01:19cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:01:21' 2026-02-22T22:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:01:26.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_remove_data_pool'\'']/last_update' 2026-02-22T22:01:26.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:01:19' 2026-02-22T22:01:26.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_remove_data_pool'\'']/state' 2026-02-22T22:01:26.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:01:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:01:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:01:26.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:01:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for cluster in ${CLUSTER1} ${CLUSTER2} 2026-02-22T22:01:26.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:01:26.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster1 osd pool rm pool_to_remove pool_to_remove --yes-i-really-really-mean-it 2026-02-22T22:01:27.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'pool_to_remove' does not exist 2026-02-22T22:01:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for cluster in ${CLUSTER1} ${CLUSTER2} 2026-02-22T22:01:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:01:27.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster2 osd pool rm pool_to_remove pool_to_remove --yes-i-really-really-mean-it 2026-02-22T22:01:28.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'pool_to_remove' does not exist 2026-02-22T22:01:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test_remove_data_pool 2026-02-22T22:01:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:28.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:01:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_remove_data_pool 2026-02-22T22:01:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:28.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_remove_data_pool 2026-02-22T22:01:28.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_remove_data_pool 2026-02-22T22:01:28.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 100% complete...done. 2026-02-22T22:01:28.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test_remove_data_pool deleted 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test_remove_data_pool 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:28.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_remove_data_pool 2026-02-22T22:01:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test_remove_data_pool 2026-02-22T22:01:28.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:01:28.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test_remove_data_pool: (2) No such file or directory 2026-02-22T22:01:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:01:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:01:28.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test_remove_data_pool deleted '' 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_remove_data_pool 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:01:28.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:01:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test_remove_data_pool 2026-02-22T22:01:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:01:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:01:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_remove_data_pool 2026-02-22T22:01:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test_remove_data_pool 2026-02-22T22:01:28.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:01:28.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test_remove_data_pool: (2) No such file or directory 2026-02-22T22:01:28.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:01:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:01:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:01:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in 0 1 2 4 8 8 8 8 16 16 2026-02-22T22:01:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:28.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:28.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:28.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:28.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:28.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:28.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:28.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T22:01:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "pool_to_remove/test", 2026-02-22T22:01:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Stopped" 2026-02-22T22:01:28.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T22:01:28.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in 0 1 2 4 8 8 8 8 16 16 2026-02-22T22:01:28.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T22:01:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T22:01:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:29.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:29.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:29.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:29.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T22:01:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "pool_to_remove/test", 2026-02-22T22:01:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Stopped" 2026-02-22T22:01:29.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T22:01:29.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in 0 1 2 4 8 8 8 8 16 16 2026-02-22T22:01:29.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:01:31.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T22:01:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T22:01:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:31.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:31.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:31.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:31.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:31.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:31.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:31.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:31.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:31.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:31.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:31.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:31.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:{ 2026-02-22T22:01:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "name": "pool_to_remove/test", 2026-02-22T22:01:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Stopped" 2026-02-22T22:01:31.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:} 2026-02-22T22:01:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:01:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in 0 1 2 4 8 8 8 8 16 16 2026-02-22T22:01:31.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:35.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:01:35.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:35.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:35.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:35.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:35.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:35.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:35.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:35.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:35.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:35.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:35.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:35.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:35.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:35.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:01:35.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:35.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:35.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:35.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:35.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:35.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:35.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:35.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:35.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:35.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:35.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:35.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:35.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:35.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:01:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:35.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:35.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:35.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:35.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:35.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:01:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:01:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:35.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:35.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:01:35.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:35.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:36.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:36.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:36.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:36.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:01:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:37.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:37.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:37.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:37.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:37.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:37.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:37.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:37.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:37.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:01:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:37.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:37.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:37.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:37.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:01:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:37.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:37.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:37.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:01:37.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:37.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:37.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:01:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:37.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:37.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:01:37.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:37.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:37.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:01:37.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:37.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:37.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:37.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:01:39.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:39.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:39.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:39.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:39.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:39.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:39.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:39.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:39.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:01:39.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:39.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:01:39.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:39.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:39.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:39.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:39.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:01:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:39.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:39.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:39.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:39.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:39.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:39.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:39.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:01:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:39.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:39.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:39.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:39.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:39.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:39.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:39.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:39.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:39.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:39.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:39.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:39.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:01:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:39.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:39.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:39.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:01:43.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:43.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:43.486 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:43.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:43.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:43.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:43.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:43.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:01:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:43.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:01:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:01:43.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:43.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:01:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:01:43.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:01:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:01:43.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:43.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:43.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:43.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:01:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:43.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:43.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:01:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:43.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:43.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:43.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:01:43.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:43.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:43.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:01:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:43.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:43.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:43.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:43.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:43.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:43.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:01:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:51.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:51.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:51.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:51.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:51.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:51.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:51.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:51.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:51.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:51.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:51.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:01:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:01:51.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:51.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:51.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:01:51.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:01:51.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:01:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:01:51.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:51.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:51.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:01:51.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:51.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:01:51.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:01:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:01:51.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:51.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:51.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:51.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:01:51.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:01:59.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:01:59.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:01:59.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:01:59.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:01:59.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:01:59.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:01:59.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:01:59.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:01:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:01:59.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:01:59.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:01:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:01:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:01:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:01:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:01:59.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:59.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:01:59.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:01:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:01:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:01:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:01:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:01:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:02:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:00.044 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:00.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:02:00.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:02:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:00.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:00.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:00.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:02:00.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:00.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:00.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:02:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:02:00.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:02:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:00.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:00.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:00.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:02:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:02:08.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:02:08.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:08.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:08.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:02:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:02:08.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:02:08.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:08.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:08.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:08.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:08.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:08.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:08.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:02:08.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:02:08.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:02:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:08.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:08.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:02:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:08.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:02:08.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:08.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:02:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:02:08.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:02:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:08.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:08.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:08.326 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:02:08.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:08.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:02:08.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:02:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:08.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:08.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:08.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:08.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:08.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:02:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:02:16.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:02:16.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:02:16.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:02:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:16.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:16.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:16.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:16.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:02:16.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:02:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:16.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:16.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:02:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:16.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:02:16.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:02:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:16.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:16.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:16.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:02:16.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:16.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:02:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:02:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:02:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:16.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:16.601 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:16.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:02:24.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:02:24.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:24.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:24.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:24.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:24.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:02:24.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:02:24.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:02:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:24.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:24.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:24.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:02:24.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:02:24.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:24.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:24.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:24.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:24.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:24.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:02:24.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:02:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:24.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:24.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:02:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:24.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:02:24.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:02:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:24.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:24.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:24.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:02:32.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:02:32.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:02:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:02:32.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:32.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:02:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:02:32.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:02:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:32.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:32.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:32.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:32.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:02:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:02:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:32.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:02:32.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:02:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:32.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:32.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:32.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:32.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:32.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:02:32.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:02:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:32.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:32.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:32.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:32.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:02:32.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:02:32.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:02:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:33.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:02:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:02:41.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:02:41.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:02:41.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:02:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:41.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:41.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:41.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:02:41.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:02:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:41.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:41.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:41.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:41.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:41.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:02:41.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:02:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:41.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:41.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:41.221 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:02:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:02:41.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:02:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:41.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:41.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:41.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:41.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:02:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:02:49.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:02:49.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:02:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:02:49.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:02:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:02:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:02:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:02:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:02:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:02:49.290 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:49.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:49.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:02:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:02:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:02:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:02:49.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:02:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:02:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:49.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:02:49.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:02:49.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:02:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:02:49.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:49.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:49.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:02:49.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:02:49.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:02:49.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:02:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:02:49.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:02:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:02:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:02:49.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:02:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:02:49.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 16 2026-02-22T22:03:05.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:05.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:05.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:05.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:05.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:05.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:05.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:05.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:05.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:05.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:05.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:05.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:05.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:05.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:05.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:05.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:05.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:05.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:05.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:05.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:05.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:05.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:05.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 16 2026-02-22T22:03:21.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:21.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:21.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:21.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:21.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:21.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:21.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:21.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:21.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:21.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:21.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:21.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:21.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:21.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:21.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:21.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:21.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:21.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:21.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:21.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:21.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:21.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:21.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:21.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:21.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:21.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:21.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:21.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:21.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:21.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:21.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:21.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:21.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:21.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:03:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:03:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster2 2026-02-22T22:03:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:21.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:03:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster2 2026-02-22T22:03:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:03:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:03:21.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:21.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:21.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:21.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:21.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:21.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:21.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:22.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:22.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:22.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:22.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:22.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:22.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:22.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:22.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:22.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:22.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:22.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:22.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:22.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:22.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:22.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:22.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:22.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:22.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:22.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:22.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:22.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:22.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:22.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:22.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:22.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:03:23.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:23.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:23.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:23.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:23.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:23.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:23.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:23.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:23.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:23.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:23.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:23.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:23.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:23.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:23.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:23.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:23.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:23.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:23.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:23.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:23.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:23.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:23.293 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:23.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:23.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:23.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:23.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:23.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:23.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:23.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:23.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:23.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:23.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:23.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:23.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:23.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:23.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:23.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:23.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:03:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:25.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:25.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:25.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:25.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:25.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:25.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:25.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:25.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:25.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:25.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:25.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:25.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:25.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:25.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:25.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:25.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:25.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:25.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:25.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:25.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:25.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:25.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:25.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:25.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:25.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:25.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:03:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:29.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:29.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:29.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:29.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:29.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:29.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:29.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:29.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:29.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:29.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:29.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:29.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:29.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:29.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:29.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:29.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:29.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:29.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:29.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:29.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:29.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:29.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:29.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:29.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:29.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:29.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:29.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:29.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:03:37.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:37.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:37.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:37.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:37.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:37.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:37.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:37.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:37.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:37.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:37.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:37.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:37.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:37.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:37.952 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:37.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:37.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:37.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:37.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:37.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:37.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:37.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:38.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:38.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:38.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:38.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:38.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:38.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:38.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:38.018 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:38.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:38.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:38.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:03:46.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:46.062 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:46.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:46.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:46.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:46.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:46.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:46.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:46.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:46.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:46.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:46.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:46.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:46.174 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:46.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:46.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:46.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:46.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:46.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:46.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:46.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:46.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:46.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:46.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:46.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:46.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:46.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:46.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:03:54.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:03:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:03:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:03:54.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:03:54.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:03:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:03:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:54.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:03:54.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:54.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:03:54.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:03:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:03:54.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:54.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:03:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:03:54.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:54.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:03:54.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:03:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:03:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:54.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:54.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:54.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:03:54.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:03:54.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:03:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:03:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:03:54.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:54.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:54.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:03:54.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:03:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:03:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:03:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:03:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:03:54.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:04:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:04:02.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:04:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:04:02.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:02.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:04:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:04:02.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:04:02.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:02.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:02.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:02.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:04:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:04:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:02.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:04:02.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:04:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:02.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:02.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:04:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:02.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:02.629 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:02.630 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:04:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:04:02.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:04:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:02.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:02.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:02.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:02.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:04:02.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:04:02.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:02.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:02.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:02.694 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:02.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:04:02.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:04:10.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:04:10.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:04:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:04:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:10.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:10.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:10.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:10.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:04:10.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:04:10.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:04:10.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:10.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:10.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:04:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:10.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:04:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:04:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:04:10.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:04:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:04:10.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:04:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:04:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:04:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:10.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:10.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:10.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:10.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:10.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:10.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:04:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:04:10.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:04:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:10.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:10.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:10.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:10.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:04:10.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:04:10.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:04:10.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:10.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:10.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:04:10.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:10.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:04:10.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:04:10.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:04:10.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:10.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:10.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:04:10.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:04:18.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:04:18.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:04:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:04:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:04:18.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:04:18.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:04:18.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:04:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:19.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:19.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:04:19.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:04:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:19.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:19.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:04:19.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:04:19.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:04:19.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:19.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:19.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:19.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:19.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:04:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:04:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:19.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:04:19.127 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:19.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:04:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:04:19.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:04:19.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:19.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:19.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:19.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:19.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:19.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:19.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:04:19.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:04:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:04:27.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:04:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:27.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:27.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:04:27.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:04:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:27.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:27.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:04:27.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:04:27.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:27.242 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:04:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:04:27.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:04:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:27.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:27.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:27.291 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:04:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:04:27.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:27.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:04:27.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:27.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:27.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:27.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:27.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:27.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:04:27.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:04:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:27.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:27.359 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:27.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:04:27.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:04:35.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:04:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:04:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:04:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:04:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:04:35.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:04:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:35.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:35.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:35.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:35.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:04:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:35.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:04:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:04:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:04:35.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:04:35.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:04:35.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:04:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:35.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:35.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:35.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:35.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:04:35.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:04:35.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:04:35.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:35.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:35.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:35.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:35.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:35.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:35.576 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:04:35.577 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:04:35.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:04:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:35.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:35.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:35.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:04:35.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 16 2026-02-22T22:04:51.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:04:51.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:04:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:51.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:51.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:04:51.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:04:51.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:04:51.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:51.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:51.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:04:51.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:04:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:04:51.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:51.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:51.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:51.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:51.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:04:51.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:04:51.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:04:51.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:51.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:51.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:04:51.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:04:51.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:04:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:04:51.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:51.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:51.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:04:51.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:04:51.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:04:51.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:04:51.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:04:51.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:04:51.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:04:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:04:51.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:04:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:04:51.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 16 2026-02-22T22:05:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:05:07.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:05:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:0 rbd mirror status pool_to_remove/test 2026-02-22T22:05:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:05:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:07.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:07.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:05:07.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:07.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:05:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:07.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status pool_to_remove/test 2026-02-22T22:05:07.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:05:07.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:07.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:1 rbd mirror status pool_to_remove/test 2026-02-22T22:05:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:05:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:05:07.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:05:07.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:07.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:05:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:05:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:05:07.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:05:07.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status pool_to_remove/test 2026-02-22T22:05:07.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:05:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:07.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:07.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:07.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:2 rbd mirror status pool_to_remove/test 2026-02-22T22:05:07.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:05:07.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:05:07.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:05:07.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:05:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:05:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:05:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:07.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:05:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:05:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:05:07.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:05:07.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status pool_to_remove/test 2026-02-22T22:05:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:05:08.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:08.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:08.026 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster2:3 rbd mirror status pool_to_remove/test 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:05:08.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:05:08.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:08.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:05:08.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:05:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:05:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:05:08.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status pool_to_remove/test 2026-02-22T22:05:08.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['pool_to_remove/test'] 2026-02-22T22:05:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:08.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:05:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: snapshot rename' 2026-02-22T22:05:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:05:08.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:05:08 TEST: snapshot rename 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:05:08 TEST: snapshot rename 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ snap_name=snap_rename 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test2 snap_rename_0 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_0 2026-02-22T22:05:08.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test2@snap_rename_0 2026-02-22T22:05:08.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:05:08.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 1 20 2026-02-22T22:05:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 1 - 1 2026-02-22T22:05:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_0 snap_rename_1 2026-02-22T22:05:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_0 2026-02-22T22:05:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_1 2026-02-22T22:05:08.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_0 mirror/test2@snap_rename_1 2026-02-22T22:05:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 2 - 1 2026-02-22T22:05:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_1 snap_rename_2 2026-02-22T22:05:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_1 2026-02-22T22:05:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_2 2026-02-22T22:05:08.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_1 mirror/test2@snap_rename_2 2026-02-22T22:05:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 3 - 1 2026-02-22T22:05:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_2 snap_rename_3 2026-02-22T22:05:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_2 2026-02-22T22:05:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_3 2026-02-22T22:05:08.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_2 mirror/test2@snap_rename_3 2026-02-22T22:05:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 4 - 1 2026-02-22T22:05:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_3 snap_rename_4 2026-02-22T22:05:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_3 2026-02-22T22:05:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_4 2026-02-22T22:05:08.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_3 mirror/test2@snap_rename_4 2026-02-22T22:05:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 5 - 1 2026-02-22T22:05:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_4 snap_rename_5 2026-02-22T22:05:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_4 2026-02-22T22:05:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_5 2026-02-22T22:05:08.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_4 mirror/test2@snap_rename_5 2026-02-22T22:05:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 6 - 1 2026-02-22T22:05:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_5 snap_rename_6 2026-02-22T22:05:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_5 2026-02-22T22:05:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_6 2026-02-22T22:05:08.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_5 mirror/test2@snap_rename_6 2026-02-22T22:05:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 7 - 1 2026-02-22T22:05:08.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_6 snap_rename_7 2026-02-22T22:05:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_6 2026-02-22T22:05:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_7 2026-02-22T22:05:08.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_6 mirror/test2@snap_rename_7 2026-02-22T22:05:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 8 - 1 2026-02-22T22:05:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_7 snap_rename_8 2026-02-22T22:05:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_7 2026-02-22T22:05:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_8 2026-02-22T22:05:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_7 mirror/test2@snap_rename_8 2026-02-22T22:05:08.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 9 - 1 2026-02-22T22:05:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_8 snap_rename_9 2026-02-22T22:05:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_8 2026-02-22T22:05:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_9 2026-02-22T22:05:08.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_8 mirror/test2@snap_rename_9 2026-02-22T22:05:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 10 - 1 2026-02-22T22:05:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_9 snap_rename_10 2026-02-22T22:05:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_9 2026-02-22T22:05:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_10 2026-02-22T22:05:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_9 mirror/test2@snap_rename_10 2026-02-22T22:05:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 11 - 1 2026-02-22T22:05:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_10 snap_rename_11 2026-02-22T22:05:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_10 2026-02-22T22:05:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_11 2026-02-22T22:05:08.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_10 mirror/test2@snap_rename_11 2026-02-22T22:05:08.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 12 - 1 2026-02-22T22:05:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_11 snap_rename_12 2026-02-22T22:05:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_11 2026-02-22T22:05:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_12 2026-02-22T22:05:08.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_11 mirror/test2@snap_rename_12 2026-02-22T22:05:08.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:08.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 13 - 1 2026-02-22T22:05:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_12 snap_rename_13 2026-02-22T22:05:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_12 2026-02-22T22:05:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_13 2026-02-22T22:05:08.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_12 mirror/test2@snap_rename_13 2026-02-22T22:05:09.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:09.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 14 - 1 2026-02-22T22:05:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_13 snap_rename_14 2026-02-22T22:05:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_13 2026-02-22T22:05:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_14 2026-02-22T22:05:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_13 mirror/test2@snap_rename_14 2026-02-22T22:05:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:09.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 15 - 1 2026-02-22T22:05:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_14 snap_rename_15 2026-02-22T22:05:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_14 2026-02-22T22:05:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_15 2026-02-22T22:05:09.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_14 mirror/test2@snap_rename_15 2026-02-22T22:05:09.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 16 - 1 2026-02-22T22:05:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_15 snap_rename_16 2026-02-22T22:05:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_15 2026-02-22T22:05:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_16 2026-02-22T22:05:09.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_15 mirror/test2@snap_rename_16 2026-02-22T22:05:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:09.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 17 - 1 2026-02-22T22:05:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_16 snap_rename_17 2026-02-22T22:05:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_16 2026-02-22T22:05:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_17 2026-02-22T22:05:09.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_16 mirror/test2@snap_rename_17 2026-02-22T22:05:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:09.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 18 - 1 2026-02-22T22:05:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_17 snap_rename_18 2026-02-22T22:05:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_17 2026-02-22T22:05:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_18 2026-02-22T22:05:09.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_17 mirror/test2@snap_rename_18 2026-02-22T22:05:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 19 - 1 2026-02-22T22:05:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_18 snap_rename_19 2026-02-22T22:05:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_18 2026-02-22T22:05:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_19 2026-02-22T22:05:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_18 mirror/test2@snap_rename_19 2026-02-22T22:05:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in `seq 1 20` 2026-02-22T22:05:09.255 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ expr 20 - 1 2026-02-22T22:05:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rename_snapshot cluster2 mirror test2 snap_rename_19 snap_rename_20 2026-02-22T22:05:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap_rename_19 2026-02-22T22:05:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local new_snap=snap_rename_20 2026-02-22T22:05:09.256 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap rename mirror/test2@snap_rename_19 mirror/test2@snap_rename_20 2026-02-22T22:05:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:05:09.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test2 snap_rename_20 2026-02-22T22:05:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap_rename_20 2026-02-22T22:05:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:05:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:05:09.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:10.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T22:05:10.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:05:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:05:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:05:10.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:05:12.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T22:05:12.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:05:12.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:05:12.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:05:12.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:05:16.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T22:05:16.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:05:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:05:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:05:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:05:24.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T22:05:24.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:error setting snapshot context: (2) No such file or directory 2026-02-22T22:05:24.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ continue 2026-02-22T22:05:24.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:05:24.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:05:32.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test2@snap_rename_20 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test2': 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 3 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 10e2379d1612 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.10e2379d1612 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:05:32.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:00:55 2026 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:00:55 2026 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:00:55 2026 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: False 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 10e2379d1612 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: 2e161c9c-920b-46da-b7ab-3890d84dfe0e 2026-02-22T22:05:32.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:05:32.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:32.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ unprotect_snapshot cluster2 mirror test4 snap1 2026-02-22T22:05:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:05:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:05:32.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap1 2026-02-22T22:05:32.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ unprotect_snapshot cluster2 mirror test4 snap2 2026-02-22T22:05:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:05:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap2 2026-02-22T22:05:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap unprotect mirror/test4@snap2 2026-02-22T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image4} 2026-02-22T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test2 2026-02-22T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:32.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test2 2026-02-22T22:05:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:32.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test2 2026-02-22T22:05:32.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test2 2026-02-22T22:05:35.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing all snapshots: 33% complete... Removing all snapshots: 66% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T22:05:35.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test2 2026-02-22T22:05:35.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for i in ${image2} ${image4} 2026-02-22T22:05:35.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test4 2026-02-22T22:05:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:05:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:35.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test4 2026-02-22T22:05:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test4 2026-02-22T22:05:35.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test4 2026-02-22T22:05:37.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T22:05:37.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test4 2026-02-22T22:05:37.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:05:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:37.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T22:05:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: demote image while daemon is offline' 2026-02-22T22:05:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:05:37.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:05:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:05:37 TEST: demote image while daemon is offline 2026-02-22T22:05:37.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:05:37 TEST: demote image while daemon is offline 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image=test_demote_image 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test_demote_image journal 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:37.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test_demote_image 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:05:37.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/test_demote_image 2026-02-22T22:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:05:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:05:37.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:05:37.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:05:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:05:37.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T22:05:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_demote_image 2026-02-22T22:05:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:37.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:37.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test_demote_image stopped 2026-02-22T22:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:05:38.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:05:38.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:05:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:05:38.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:38.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:38.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:05:38.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:05:38.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:38.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:38.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:38.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:38.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:38.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:05:38.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:05:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:05:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:05:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:05:38.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:38.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:38.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:38.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:38.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:2 2026-02-22T22:05:38.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:38.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:38.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:05:38.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:2 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:2 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:05:38.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:05:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:05:38.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:05:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:05:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:05:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:05:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:38.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:38.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:3 2026-02-22T22:05:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:05:38.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:3 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:38.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:3 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:05:38.670 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:05:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:05:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:05:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:05:38.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:38.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:38.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:38.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:05:39.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:05:39.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:39.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:05:39.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:05:39.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:05:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:05:39.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:39.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:05:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:05:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:05:39.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_demote_image 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:39.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:05:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:05:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:39.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_demote_image started 2026-02-22T22:05:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:05:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:05:40.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:05:40.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:05:40.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:05:40.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:40.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:05:40.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:05:40.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:40.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:40.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:40.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:05:40.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:05:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:05:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:05:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:05:40.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:40.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:40.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:40.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:05:40.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:40.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:40.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:05:40.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:05:40.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:05:40.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:05:40.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:05:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:05:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:05:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:05:40.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:40.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:40.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:40.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:05:40.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:05:40.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:05:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:05:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:05:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:05:40.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:41.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_demote_image'] 2026-02-22T22:05:41.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:05:41.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:41.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:05:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:05:42.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:05:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_demote_image 2026-02-22T22:05:42.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:42.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:05:42.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:42.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:05:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:05:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:05:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:05:42.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:05:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:05:42.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:05:42.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:05:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:05:42.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:05:42.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:42.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:05:42.080 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_demote_image 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_demote_image ']' 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_demote_image' 2026-02-22T22:05:42.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_demote_image 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:42.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_demote_image 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:05:42.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:42.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:42.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:05:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_demote_image 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_demote_image 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_demote_image '' 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_demote_image 2026-02-22T22:05:42.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:05:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:05:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:05:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:05:42.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=99], [object_number=2, tag_tid=1, entry_tid=98]]], state=connected] 2026-02-22T22:05:42.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:05:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_demote_image 2026-02-22T22:05:42.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_demote_image '..*' 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_demote_image 2026-02-22T22:05:42.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:05:42.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:05:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:05:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:05:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=99], [object_number=2, tag_tid=1, entry_tid=98]]], state=connected] 2026-02-22T22:05:42.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' '' 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' = '' 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:05:42.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:05:42.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=2 2026-02-22T22:05:42.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:05:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:05:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:05:42.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=2 2026-02-22T22:05:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:05:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:05:42.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:05:42.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=3 2026-02-22T22:05:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:05:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:05:42.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:05:42.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=1 2026-02-22T22:05:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 2 = 2 -a 3 -le 1 2026-02-22T22:05:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:05:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:05:42.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_demote_image 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_demote_image ']' 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_demote_image' 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_demote_image 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:05:42.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:05:42.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_demote_image 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:05:42.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:42.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:05:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:42.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_demote_image 2026-02-22T22:05:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_demote_image 2026-02-22T22:05:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_demote_image '' 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_demote_image 2026-02-22T22:05:42.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:05:42.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:05:42.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_demote_image 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_demote_image '..*' 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_demote_image 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_demote_image 2026-02-22T22:05:42.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:05:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:05:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:05:42.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:05:42.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:05:42.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+replaying 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:42.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:42.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+replaying '' '' 2026-02-22T22:05:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:05:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:43.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:43.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:43.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:43.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:41 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:05:43.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:05:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:41' 2026-02-22T22:05:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:05:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:05:43.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:05:43.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:05:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:05:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:05:43.779 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:05:43.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:05:43.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:05:43.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:05:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:05:43.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:05:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:41' 2026-02-22T22:05:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+replaying '2026-02-22 22:05:41' 2026-02-22T22:05:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:43.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:05:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:05:41' 2026-02-22T22:05:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:05:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK243184322mirror.0trial10419.2.3-689-gf90f4eaetrueOK43244328mirror.1trial10419.2.3-689-gf90f4eaefalseOK43304335mirror.2trial10419.2.3-689-gf90f4eaefalseOK43394344mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}43184322mirror.0trial1042026-02-22 22:05:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:05:41test_demote_imagebc5fbfbf-03de-4873-8b33-4df3be2f4253up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}43184322mirror.0trial1042026-02-22 22:05:41cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:05:41' 2026-02-22T22:05:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:05:43.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T22:05:43.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:41' 2026-02-22T22:05:43.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T22:05:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:05:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:05:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:05:43.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:05:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+stopped 2026-02-22T22:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:05:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+stopped '' '' 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:44.821 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T22:05:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:05:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:05:44.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:41 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:41' 2026-02-22T22:05:44.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:05:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T22:05:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:05:44.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T22:05:44.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:05:44.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:05:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:05:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:05:44.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:05:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:05:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:05:44.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:41' 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+stopped '2026-02-22 22:05:41' 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:05:41' 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:44.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:05:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK245054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 22:05:41cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}2026-02-22 22:05:16test_demote_imagebc5fbfbf-03de-4873-8b33-4df3be2f4253up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 22:05:41cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}2026-02-22 22:05:41' 2026-02-22T22:05:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:05:44.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T22:05:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:41' 2026-02-22T22:05:44.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T22:05:44.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T22:05:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:05:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T22:05:44.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:05:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster1 2026-02-22T22:05:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:44.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:05:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:0 2026-02-22T22:05:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:05:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:05:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:05:44.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:05:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=50528 2026-02-22T22:05:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 50528 ']' 2026-02-22T22:05:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 50528 2026-02-22T22:05:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:44.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50528 '$2 == pid {print; exit 1}' 2026-02-22T22:05:44.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 50528 0.2 0.0 835376 74596 ? Ssl 21:46 0:03 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:44.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:44.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:45.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:45.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50528 '$2 == pid {print; exit 1}' 2026-02-22T22:05:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:05:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:45.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50528 '$2 == pid {print; exit 1}' 2026-02-22T22:05:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:05:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:05:45.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:45.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:45.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:05:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:45.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:45.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:45.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:45.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:05:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:45.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:05:45.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:05:45.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:05:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:05:45.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:05:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T22:05:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:05:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:05:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:45.989 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:05:45.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:05:45.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:05:46.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:05:46.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:05:46.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=50571 2026-02-22T22:05:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 50571 ']' 2026-02-22T22:05:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 50571 2026-02-22T22:05:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:46.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50571 '$2 == pid {print; exit 1}' 2026-02-22T22:05:46.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 50571 0.0 0.0 698088 60984 ? Ssl 21:46 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:46.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:47.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:47.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50571 '$2 == pid {print; exit 1}' 2026-02-22T22:05:47.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:47.025 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50571 '$2 == pid {print; exit 1}' 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:05:47.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:47.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:05:47.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:05:47.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:47.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:05:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:05:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:47.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:05:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:05:47.061 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:05:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:05:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:05:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:05:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:05:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:05:47.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:05:47.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:05:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:2 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:47.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:05:47.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:05:47.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:05:47.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:05:47.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:05:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:05:47.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:05:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=50633 2026-02-22T22:05:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 50633 ']' 2026-02-22T22:05:47.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 50633 2026-02-22T22:05:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:47.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50633 '$2 == pid {print; exit 1}' 2026-02-22T22:05:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 50633 0.0 0.0 698092 57572 ? Ssl 21:46 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:47.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:47.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:48.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:48.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50633 '$2 == pid {print; exit 1}' 2026-02-22T22:05:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:05:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:48.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50633 '$2 == pid {print; exit 1}' 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:48.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:05:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:05:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:48.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:05:48.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:05:48.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:05:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:05:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:05:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:05:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:48.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:05:48.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:05:48.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:05:48.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:05:48.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:05:48.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:05:48.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:05:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:05:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:05:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T22:05:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:05:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:05:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:48.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:3 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:05:48.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:05:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:05:48.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:05:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=50700 2026-02-22T22:05:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 50700 ']' 2026-02-22T22:05:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 50700 2026-02-22T22:05:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:48.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:05:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:48.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50700 '$2 == pid {print; exit 1}' 2026-02-22T22:05:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 50700 0.0 0.0 698092 58008 ? Ssl 21:46 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:05:48.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:49.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:49.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50700 '$2 == pid {print; exit 1}' 2026-02-22T22:05:49.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:05:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:05:49.200 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=50700 '$2 == pid {print; exit 1}' 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:05:49.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:05:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:49.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:05:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:05:49.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:05:49.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:49.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:05:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:05:49.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:05:49.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:05:49.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:05:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:05:49.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_demote_image 100 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:05:49.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_demote_image --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:05:49.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:05:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:05:49.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T22:05:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster2 mirror test_demote_image 2026-02-22T22:05:49.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image demote mirror/test_demote_image 2026-02-22T22:05:49.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T22:05:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster1 2026-02-22T22:05:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:49.352 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:05:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:0 2026-02-22T22:05:49.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:49.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:49.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:2 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:49.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:49.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:3 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:05:49.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:05:49.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror test_demote_image 2026-02-22T22:05:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:05:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test_demote_image.status 2026-02-22T22:05:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test_demote_image.status 2026-02-22T22:05:49.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:49.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test_demote_image.status 2026-02-22T22:05:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test_demote_image.status 2026-02-22T22:05:49.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:49.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_status_log=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.status 2026-02-22T22:05:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ get_newest_mirror_snapshot cluster2 mirror test_demote_image /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.status 2026-02-22T22:05:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap list --all mirror/test_demote_image --format xml 2026-02-22T22:05:49.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:05:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:05:49.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:05:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:-:1.1: Document is empty 2026-02-22T22:05:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T22:05:49.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:^ 2026-02-22T22:05:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snapshot_id= 2026-02-22T22:05:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:05:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:05:49.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:05:49.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ get_newest_mirror_snapshot cluster1 mirror test_demote_image /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.status 2026-02-22T22:05:49.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:49.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:49.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:49.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local log=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.status 2026-02-22T22:05:49.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 snap list --all mirror/test_demote_image --format xml 2026-02-22T22:05:49.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:05:49.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:05:49.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:05:49.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:-:1.1: Document is empty 2026-02-22T22:05:49.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: 2026-02-22T22:05:49.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:^ 2026-02-22T22:05:49.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local primary_snapshot_id= 2026-02-22T22:05:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test '' = '' 2026-02-22T22:05:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+unknown 2026-02-22T22:05:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:49.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:49.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:49.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:49.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:49.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:50.706 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-22T22:05:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:50.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:50.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T22:05:50.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:05:50.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:05:50.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:46 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:05:50.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:05:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:05:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:50.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:05:52.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+unknown '' '' 2026-02-22T22:05:52.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:52.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T22:05:52.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:05:52.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:05:52.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:51 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:05:52.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:05:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T22:05:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:05:52.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T22:05:52.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:05:52.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T22:05:52.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:05:52.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T22:05:52.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:05:52.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:05:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:05:52.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:05:52.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:51' 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+unknown '2026-02-22 22:05:51' 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:05:51' 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:52.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:05:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING245054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 22:05:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9ddown+unknownstatus not foundtest_demote_imagebc5fbfbf-03de-4873-8b33-4df3be2f4253up+unknownremote image is not primary45054508mirror.0trial1042026-02-22 22:05:51cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9ddown+unknownstatus not found' 2026-02-22T22:05:52.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:05:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T22:05:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:51' 2026-02-22T22:05:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T22:05:52.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T22:05:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T22:05:52.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T22:05:52.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+unknown 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:52.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:52.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:05:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-22T22:05:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:53.816 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T22:05:53.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":101,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":5}} 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:53 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T22:05:53.849 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:51' 2026-02-22T22:05:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:05:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:05:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:53.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:05:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-22T22:05:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:55.851 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:55.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T22:05:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:05:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:05:55.884 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: stopped 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:55 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:51' 2026-02-22T22:05:55.885 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:05:55.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:05:55.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:05:55.886 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+unknown '' '' 2026-02-22T22:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:05:59.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:05:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:59.888 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T22:05:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:05:59.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:59 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+unknown 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: remote image is not primary 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:05:56' 2026-02-22T22:05:59.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+unknown' 2026-02-22T22:05:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+unknown 2026-02-22T22:05:59.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:05:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: remote image is not primary 2026-02-22T22:05:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:05:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T22:05:59.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:05:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T22:05:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:05:59.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:05:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:05:59.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:05:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:59' 2026-02-22T22:05:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+unknown '2026-02-22 22:05:59' 2026-02-22T22:05:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:05:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:59.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+unknown 2026-02-22T22:05:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:05:59' 2026-02-22T22:05:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:05:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:05:59.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:05:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING1157205725mirror.0trial10419.2.3-689-gf90f4eaetrueOK57275731mirror.1trial10419.2.3-689-gf90f4eaefalseOK57335737mirror.2trial10419.2.3-689-gf90f4eaefalseOK57425745mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}57205725mirror.0trial1042026-02-22 22:05:53cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:05:56test_demote_imagebc5fbfbf-03de-4873-8b33-4df3be2f4253up+unknownremote image is not primary57205725mirror.0trial1042026-02-22 22:05:59cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+unknownremote image is not primary2026-02-22 22:05:56' 2026-02-22T22:05:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:05:59.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T22:05:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:05:59' 2026-02-22T22:05:59.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T22:05:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+unknown 2026-02-22T22:05:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+unknown 2026-02-22T22:05:59.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+unknown 2026-02-22T22:05:59.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+unknown 2026-02-22T22:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test_demote_image 2026-02-22T22:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:05:59.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test_demote_image.export 2026-02-22T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test_demote_image.export 2026-02-22T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:59.964 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.export 2026-02-22T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test_demote_image.export 2026-02-22T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test_demote_image.export 2026-02-22T22:05:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:05:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.export 2026-02-22T22:05:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.export 2026-02-22T22:05:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test_demote_image /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.export 2026-02-22T22:06:00.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test_demote_image /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.export 2026-02-22T22:06:00.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:00.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.export 2026-02-22T22:06:00.199 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_demote_image.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test_demote_image.export 2026-02-22T22:06:00.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test_demote_image 2026-02-22T22:06:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force= 2026-02-22T22:06:00.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_demote_image 2026-02-22T22:06:00.266 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster2 mirror test_demote_image 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:00.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:00.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T22:06:01.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:01.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T22:06:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:01.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:06:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:01.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:06:01.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:01.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:06:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:06:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:01.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:06:01.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:01.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:06:01.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:06:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:06:01.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = stopped 2026-02-22T22:06:01.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:01.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:06:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test_demote_image started 2026-02-22T22:06:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:03.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:03.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_demote_image 2026-02-22T22:06:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T22:06:03.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T22:06:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:06:03.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:03.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_demote_image 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:03.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:06:03.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:06:03.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:06:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:03.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:06:03.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:03.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:03.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:03.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_demote_image 2026-02-22T22:06:03.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:03.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:06:03.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:06:03.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:03.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:06:03.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:06:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:06:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_demote_image up+stopped 2026-02-22T22:06:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:03.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:06:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:06:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:03.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_demote_image up+stopped '' '' 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:04.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:04.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_demote_image 2026-02-22T22:06:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:06:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:06:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:06:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:06:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:06:04.433 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:04 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4}} 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:01' 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T22:06:04.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:06:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T22:06:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:04.435 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:06:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:06:04.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:06:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:06:04.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:06:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:06:04.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:06:04.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:04' 2026-02-22T22:06:04.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_demote_image up+stopped '2026-02-22 22:06:04' 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:06:04' 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:04.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:06:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK257205725mirror.0trial10419.2.3-689-gf90f4eaetrueOK57275731mirror.1trial10419.2.3-689-gf90f4eaefalseOK57335737mirror.2trial10419.2.3-689-gf90f4eaefalseOK57425745mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}57205725mirror.0trial1042026-02-22 22:05:53cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:06:01test_demote_imagebc5fbfbf-03de-4873-8b33-4df3be2f4253up+stoppedlocal image is primary57205725mirror.0trial1042026-02-22 22:06:04cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4}}2026-02-22 22:06:01' 2026-02-22T22:06:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:06:04.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T22:06:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:04' 2026-02-22T22:06:04.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T22:06:04.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T22:06:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:06:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T22:06:04.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:06:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:04.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_demote_image up+replaying 2026-02-22T22:06:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:06:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:04.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:04.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:05.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_demote_image up+replaying '' '' 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:05.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_demote_image 2026-02-22T22:06:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_demote_image: 2026-02-22T22:06:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: bc5fbfbf-03de-4873-8b33-4df3be2f4253 2026-02-22T22:06:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:06:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4}} 2026-02-22T22:06:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:06:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:01 2026-02-22T22:06:05.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:06:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T22:06:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:06:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:06:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:04' 2026-02-22T22:06:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:06:05.513 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:06:05.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:06:05.514 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4}} 2026-02-22T22:06:05.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:05.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:05.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:06:05.515 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:06:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:06:05.516 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:06:05.517 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:06:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:01' 2026-02-22T22:06:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_demote_image up+replaying '2026-02-22 22:06:01' 2026-02-22T22:06:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:05.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:06:01' 2026-02-22T22:06:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:05.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:06:05.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK245054508mirror.0trial10419.2.3-689-gf90f4eaetrueOK45124515mirror.1trial10419.2.3-689-gf90f4eaefalseOK45194524mirror.2trial10419.2.3-689-gf90f4eaefalseOK45284535mirror.3trial10419.2.3-689-gf90f4eaefalseOKtesta4e8f094-9fd1-42cf-992f-29d2efd77535up+stoppedlocal image is primary45054508mirror.0trial1042026-02-22 22:06:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":0,"entries_per_second":0.0,"non_primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":118}}2026-02-22 22:05:53test_demote_imagebc5fbfbf-03de-4873-8b33-4df3be2f4253up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":1,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":0,"object_number":0,"tag_tid":4}}45054508mirror.0trial1042026-02-22 22:06:01cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 22:06:04' 2026-02-22T22:06:05.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:06:05.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/last_update' 2026-02-22T22:06:05.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:01' 2026-02-22T22:06:05.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_demote_image'\'']/state' 2026-02-22T22:06:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:06:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:05.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:06:05.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:05.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster1 mirror test_demote_image 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster1 mirror test_demote_image 2026-02-22T22:06:05.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_demote_image 2026-02-22T22:06:05.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_demote_image 2026-02-22T22:06:05.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 rm mirror/test_demote_image 2026-02-22T22:06:05.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:06:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:06:05.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: disable mirror while daemon is stopped' 2026-02-22T22:06:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:05.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:05 TEST: disable mirror while daemon is stopped 2026-02-22T22:06:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:05 TEST: disable mirror while daemon is stopped 2026-02-22T22:06:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster1 2026-02-22T22:06:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:05.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:05.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:06:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:05.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:0 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:06:05.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:05.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:06:05.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:05.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=86132 2026-02-22T22:06:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 86132 ']' 2026-02-22T22:06:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 86132 2026-02-22T22:06:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:05.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:05.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86132 '$2 == pid {print; exit 1}' 2026-02-22T22:06:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 86132 0.7 0.0 744684 49436 ? Ssl 22:05 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:06.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86132 '$2 == pid {print; exit 1}' 2026-02-22T22:06:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:06.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86132 '$2 == pid {print; exit 1}' 2026-02-22T22:06:06.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:06.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:06.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:06.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:06.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:06.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:06.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:06.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:06.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:06:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:06.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:06.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:06.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:06:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:06.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:06.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:06:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:06.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:06.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T22:06:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:06:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:06.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:06:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:1 2026-02-22T22:06:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:06:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:06.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:06:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:06:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:06.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:06:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:06.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=86175 2026-02-22T22:06:06.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 86175 ']' 2026-02-22T22:06:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 86175 2026-02-22T22:06:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:06.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86175 '$2 == pid {print; exit 1}' 2026-02-22T22:06:06.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 86175 0.4 0.0 670920 42676 ? Ssl 22:05 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:06.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:07.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:07.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86175 '$2 == pid {print; exit 1}' 2026-02-22T22:06:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86175 '$2 == pid {print; exit 1}' 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:07.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:07.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:07.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:07.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:06:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:06:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:07.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:07.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:06:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:06:07.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:07.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:07.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T22:06:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:06:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:06:07.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:2 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:07.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:06:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:06:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:07.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:06:07.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:06:07.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:06:07.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=86227 2026-02-22T22:06:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 86227 ']' 2026-02-22T22:06:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 86227 2026-02-22T22:06:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:07.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86227 '$2 == pid {print; exit 1}' 2026-02-22T22:06:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 86227 0.4 0.0 670920 42564 ? Ssl 22:05 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:07.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:08.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:08.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86227 '$2 == pid {print; exit 1}' 2026-02-22T22:06:08.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:08.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86227 '$2 == pid {print; exit 1}' 2026-02-22T22:06:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:06:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:06:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:08.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:08.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:06:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:06:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:08.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:06:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:06:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:08.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:08.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:06:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:06:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:3 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:06:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:06:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:06:08.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:06:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=86288 2026-02-22T22:06:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 86288 ']' 2026-02-22T22:06:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 86288 2026-02-22T22:06:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:08.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:08.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86288 '$2 == pid {print; exit 1}' 2026-02-22T22:06:08.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 86288 0.3 0.0 670920 45592 ? Ssl 22:05 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:08.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:09.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86288 '$2 == pid {print; exit 1}' 2026-02-22T22:06:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:09.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=86288 '$2 == pid {print; exit 1}' 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:09.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:06:09.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:09.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:09.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:06:09.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:09.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:06:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:06:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:09.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:09.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:06:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:06:10.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:06:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster2 2026-02-22T22:06:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:10.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:10.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:0 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:10.011 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:06:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:06:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:10.012 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:10.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:06:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:06:10.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:06:10.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=54475 2026-02-22T22:06:10.021 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 54475 ']' 2026-02-22T22:06:10.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 54475 2026-02-22T22:06:10.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:10.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:10.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:10.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54475 '$2 == pid {print; exit 1}' 2026-02-22T22:06:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 54475 0.1 0.0 772888 66852 ? Ssl 21:48 0:02 rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:10.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:11.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:11.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54475 '$2 == pid {print; exit 1}' 2026-02-22T22:06:11.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:11.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54475 '$2 == pid {print; exit 1}' 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:11.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:11.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:06:11.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:11.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:11.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:11.073 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:06:11.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:11.083 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:0 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:11.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:06:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:06:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:11.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:06:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:06:11.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:06:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T22:06:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:06:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:11.097 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:1 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:1 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:1 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:06:11.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:06:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:11.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T22:06:11.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:06:11.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:06:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=54519 2026-02-22T22:06:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 54519 ']' 2026-02-22T22:06:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 54519 2026-02-22T22:06:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:11.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:11.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54519 '$2 == pid {print; exit 1}' 2026-02-22T22:06:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 54519 0.0 0.0 599740 56860 ? Rsl 21:48 0:00 rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:11.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:12.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:12.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54519 '$2 == pid {print; exit 1}' 2026-02-22T22:06:12.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:12.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54519 '$2 == pid {print; exit 1}' 2026-02-22T22:06:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-22T22:06:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:06:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:06:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:06:12.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:12.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:06:12.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:06:12.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:12.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:12.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:06:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:06:12.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:1 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:1 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:12.172 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:06:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:06:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:12.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T22:06:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:06:12.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:06:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T22:06:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:06:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:12.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:2 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:2 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:2 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T22:06:12.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:06:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:06:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:12.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T22:06:12.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:06:12.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:06:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=54580 2026-02-22T22:06:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 54580 ']' 2026-02-22T22:06:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 54580 2026-02-22T22:06:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:12.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:12.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54580 '$2 == pid {print; exit 1}' 2026-02-22T22:06:12.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 54580 0.0 0.0 698096 59744 ? Ssl 21:48 0:00 rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:12.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:12.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:13.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:13.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54580 '$2 == pid {print; exit 1}' 2026-02-22T22:06:13.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:13.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54580 '$2 == pid {print; exit 1}' 2026-02-22T22:06:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-22T22:06:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:06:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:06:13.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:13.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:06:13.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:06:13.245 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:06:13.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:06:13.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:13.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:06:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:06:13.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:06:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:2 2026-02-22T22:06:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:2 2026-02-22T22:06:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:13.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:13.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:13.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T22:06:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:06:13.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:06:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T22:06:13.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:3 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:3 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:3 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:13.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:06:13.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T22:06:13.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:06:13.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:06:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=54657 2026-02-22T22:06:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 54657 ']' 2026-02-22T22:06:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 54657 2026-02-22T22:06:13.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:13.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54657 '$2 == pid {print; exit 1}' 2026-02-22T22:06:13.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 54657 0.0 0.0 698096 57504 ? Ssl 21:48 0:00 rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:13.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:14.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:14.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54657 '$2 == pid {print; exit 1}' 2026-02-22T22:06:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:14.309 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=54657 '$2 == pid {print; exit 1}' 2026-02-22T22:06:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-22T22:06:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:06:14.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:14.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:14.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:06:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:06:14.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:06:14.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:06:14.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:06:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:06:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:14.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:06:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:06:14.343 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:06:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:3 2026-02-22T22:06:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:3 2026-02-22T22:06:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:06:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:06:14.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:14.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T22:06:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:06:14.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:06:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:14.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_pool_mirror_mode cluster2 mirror image 2026-02-22T22:06:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=image 2026-02-22T22:06:14.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror image 2026-02-22T22:06:14.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:note: changing mirroring mode from pool to image 2026-02-22T22:06:14.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ disable_mirror cluster2 mirror test 2026-02-22T22:06:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:14.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/test 2026-02-22T22:06:14.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Mirroring disabled 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:14.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:14.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e269fced7d 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e269fced7d 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 10e269fced7d 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster1 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:14.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:0 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:14.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:14.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:1 2026-02-22T22:06:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:06:14.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:14.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:14.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:14.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:2 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:14.489 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:14.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:06:14.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:14.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:14.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:3 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:14.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:06:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:06:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:14.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:14.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test deleted 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:14.528 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:14.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:14.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id=10e269fced7d 2026-02-22T22:06:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:14.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 10e269fced7d 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e269fced7d 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:14.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:14.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e269fced7d 2026-02-22T22:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e269fced7d 2026-02-22T22:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 10e269fced7d -o 10e269fced7d = 10e269fced7d 2026-02-22T22:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:06:14.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:06:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:14.894 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 10e269fced7d 2026-02-22T22:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:15.895 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e269fced7d 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:15.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e269fced7d 2026-02-22T22:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e269fced7d 2026-02-22T22:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 10e269fced7d -o 10e269fced7d = 10e269fced7d 2026-02-22T22:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:06:15.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:15.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:06:17.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 10e269fced7d 2026-02-22T22:06:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e269fced7d 2026-02-22T22:06:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:17.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:17.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=10e269fced7d 2026-02-22T22:06:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 10e269fced7d 2026-02-22T22:06:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 10e269fced7d -o 10e269fced7d = 10e269fced7d 2026-02-22T22:06:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:06:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:06:17.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:17.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:06:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 10e269fced7d 2026-02-22T22:06:21.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=10e269fced7d 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:21.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:21.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:21.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:21.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_pool_mirror_mode cluster2 mirror pool 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=pool 2026-02-22T22:06:21.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror pool enable mirror pool 2026-02-22T22:06:22.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:note: changing mirroring mode from image to pool 2026-02-22T22:06:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ enable_journaling cluster2 mirror test 2026-02-22T22:06:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:22.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 feature enable mirror/test journaling 2026-02-22T22:06:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T22:06:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:22.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:06:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:22.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:22.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:22.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:22.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:22.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:22.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:06:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:22.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T22:06:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:22.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:22.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:22.201 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:22.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:06:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:22.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:23.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:23.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:23.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:23.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:23.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:23.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:23.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:06:23.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:23.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:06:25.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T22:06:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:25.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:25.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:25.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:25.238 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:25.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:25.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:06:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:25.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:29.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:29.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=16a45866ec7e 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a45866ec7e 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 16a45866ec7e 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:29.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:06:30.294 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:30.295 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:06:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:06:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:30.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:06:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:30.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:06:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:30.303 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:30.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:30.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:30.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:30.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:30.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:06:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:06:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:06:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:06:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:30.361 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:06:30.362 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:30.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:30.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:06:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:06:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test 2026-02-22T22:06:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:06:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:30.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:06:30.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:06:30.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:30.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test 2026-02-22T22:06:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:06:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test 2026-02-22T22:06:30.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:30.472 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:06:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:06:30.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:06:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:30.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test 2026-02-22T22:06:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:06:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:06:31.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:31.527 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:06:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:31.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:31.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:31.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:31.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:31.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:06:31.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:06:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:06:31.591 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:06:31.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:06:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:06:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: non-default namespace image mirroring' 2026-02-22T22:06:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:31.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:31 TEST: non-default namespace image mirroring 2026-02-22T22:06:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:31 TEST: non-default namespace image mirroring 2026-02-22T22:06:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - replay' 2026-02-22T22:06:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:31.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:31.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:31 - replay 2026-02-22T22:06:31.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:31 - replay 2026-02-22T22:06:31.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test journal 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror/ns1 test 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:31.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:31.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/ns1/test 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns2 test journal 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror/ns2 test 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:31.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:31.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/ns2/test 2026-02-22T22:06:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror/ns2 2026-02-22T22:06:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:06:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:06:31.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ enable_mirror cluster2 mirror/ns2 test journal 2026-02-22T22:06:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:06:31.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image enable mirror/ns2/test journal 2026-02-22T22:06:31.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Mirroring enabled 2026-02-22T22:06:31.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 info mirror/ns2/test 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test': 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 128 MiB in 32 objects 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 0 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 18789b08ca02 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.18789b08ca02 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:06:31.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:06:31 2026 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:06:31 2026 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:06:31 2026 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 18789b08ca02 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: 41aca810-eaf7-41b2-b651-7211a4bd4b03 2026-02-22T22:06:31.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: true 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test started 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:31.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:06:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:31.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:32.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror/ns1 test started 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:06:32.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:32.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:32.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:06:32.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:32.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:32.919 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:32.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:32.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:32.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:32.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:32.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:32.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:32.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:32.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-02-22T22:06:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:06:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:32.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:32.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:32.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:32.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:32.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:32.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:32.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:32.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:33.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:33.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-02-22T22:06:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:06:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:33.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:33.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:06:33.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:06:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:06:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:33.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:33.043 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:06:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:33.046 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:33.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-02-22T22:06:33.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:33.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:33.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:06:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:06:33.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:06:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:33.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:33.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:33.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:33.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:33.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:06:34.146 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:34.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:34.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:34.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:34.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:34.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:34.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:34.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:34.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:34.212 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:34.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:34.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:34.222 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:34.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:34.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:34.269 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:34.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:06:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:34.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:34.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:34.318 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:34.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:34.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:34.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:06:34.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:06:34.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:34.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:34.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test'] 2026-02-22T22:06:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:34.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:34.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:34.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T22:06:36.376 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:36.377 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:36.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:36.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:36.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test 2026-02-22T22:06:36.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:06:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:06:36.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:36.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:06:36.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:06:36.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror/ns2 test 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror/ns2 test started 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:06:36.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:36.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror/ns2 test started 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns2/test 2026-02-22T22:06:37.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:37.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:37.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns2/test 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:37.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:37.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:37.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:37.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:37.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns2/test 2026-02-22T22:06:37.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:06:37.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:06:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:06:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:06:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:06:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror/ns1 test 100 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:06:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:06:37.542 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:06:37.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:06:37.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 9091.09 bytes/sec: 36 MiB/s 2026-02-22T22:06:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror/ns2 test 100 2026-02-22T22:06:37.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:06:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:06:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:06:37.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/ns2/test --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:06:37.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:06:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:06:37.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 6250.13 bytes/sec: 24 MiB/s 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 test 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror/ns1 test 2026-02-22T22:06:37.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:06:37.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror/ns1 test 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/ns1/test' 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/ns1/test 2026-02-22T22:06:37.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:37.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:37.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/ns1/test 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:37.824 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:37.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:37.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:37.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/ns1/test 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror/ns1 test 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test '' 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:37.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:37.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:37.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:06:37.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T22:06:37.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test.status 2026-02-22T22:06:37.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:37.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:37.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test 2026-02-22T22:06:37.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:37.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:06:37.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:37.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror/ns1 test 2026-02-22T22:06:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test '..*' 2026-02-22T22:06:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:37.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:06:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T22:06:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test.status 2026-02-22T22:06:37.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test 2026-02-22T22:06:37.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:37.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:06:37.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:06:37.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:06:37.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=2 2026-02-22T22:06:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:06:37.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:06:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=1 2026-02-22T22:06:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:37.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:06:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:06:37.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=101 2026-02-22T22:06:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:06:37.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:06:37.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=3 2026-02-22T22:06:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:06:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:06:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:06:37.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:06:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror/ns1 test 2026-02-22T22:06:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:38.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/ns1/test' 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/ns1/test 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:38.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:38.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/ns1/test 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:38.158 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:38.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:38.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:38.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/ns1/test 2026-02-22T22:06:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:38.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror/ns1 test 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test '' 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:06:38.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:38.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test 2026-02-22T22:06:38.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:38.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:06:38.244 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror/ns1 test 2026-02-22T22:06:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.246 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test '..*' 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.247 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test 2026-02-22T22:06:38.248 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:38.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:06:38.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:38.279 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:06:38.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:06:38.280 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T22:06:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror/ns1 test 2026-02-22T22:06:38.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/ns1/test' 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/ns1/test 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:38.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:38.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/ns1/test 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:38.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:38.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:38.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:38.696 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/ns1/test 2026-02-22T22:06:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror/ns1 test 2026-02-22T22:06:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test '' 2026-02-22T22:06:38.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:06:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test 2026-02-22T22:06:38.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:38.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:38.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror/ns1 test 2026-02-22T22:06:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test '..*' 2026-02-22T22:06:38.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test.status 2026-02-22T22:06:38.777 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test 2026-02-22T22:06:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:38.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:38.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.status 2026-02-22T22:06:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns2 test 2026-02-22T22:06:38.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror/ns2 test 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:06:38.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:06:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:06:38.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:06:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror/ns2 test 2026-02-22T22:06:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:39.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test ']' 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/ns2/test' 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/ns2/test 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:39.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:39.013 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/ns2/test 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:39.014 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:39.015 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:39.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:39.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/ns2/test 2026-02-22T22:06:39.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror/ns2 test 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns2 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns2 test '' 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns2 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:39.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:06:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns2-test.status 2026-02-22T22:06:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns2-test.status 2026-02-22T22:06:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.status 2026-02-22T22:06:39.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns2/test 2026-02-22T22:06:39.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.status 2026-02-22T22:06:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:39.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:39.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.status 2026-02-22T22:06:39.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror/ns2 test 2026-02-22T22:06:39.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns2 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns2 test '..*' 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns2 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns2-test.status 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns2-test.status 2026-02-22T22:06:39.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.status 2026-02-22T22:06:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns2/test 2026-02-22T22:06:39.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.status 2026-02-22T22:06:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:39.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:39.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.status 2026-02-22T22:06:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test 2026-02-22T22:06:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:39.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test up+replaying primary_position 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test up+replaying primary_position '' 2026-02-22T22:06:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:40.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:40.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test 2026-02-22T22:06:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T22:06:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 6623341e-df0b-4017-995f-1480389fd739 2026-02-22T22:06:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:06:40.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:35 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:06:40.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:06:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:06:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:06:40.389 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:06:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:40.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:06:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:06:40.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:06:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:06:40.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:06:40.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:35' 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test up+replaying '2026-02-22 22:06:35' 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:06:35' 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T22:06:40.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING157915796mirror.0trial10419.2.3-689-gf90f4eaetrueOK57985801mirror.1trial10419.2.3-689-gf90f4eaefalseOK58035808mirror.2trial10419.2.3-689-gf90f4eaefalseOK58105821mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest6623341e-df0b-4017-995f-1480389fd739up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}57915796mirror.0trial1042026-02-22 22:06:35cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:06:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:06:40.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:06:40.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:35' 2026-02-22T22:06:40.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:06:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:06:40.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:40.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:06:40.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:40.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns2 test 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns2 test up+replaying primary_position 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:40.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:40.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:41.431 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror/ns2 test up+replaying primary_position '' 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:41.432 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns2/test 2026-02-22T22:06:41.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 41aca810-eaf7-41b2-b651-7211a4bd4b03 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:35 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:06:41.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:06:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:06:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:06:41.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:06:41.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:06:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:06:41.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:06:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:06:41.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:06:41.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:06:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:35' 2026-02-22T22:06:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns2 test up+replaying '2026-02-22 22:06:35' 2026-02-22T22:06:41.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:06:35' 2026-02-22T22:06:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:41.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns2 --verbose --format xml 2026-02-22T22:06:41.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING157915796mirror.0trial10419.2.3-689-gf90f4eaetrueOK57985801mirror.1trial10419.2.3-689-gf90f4eaefalseOK58035808mirror.2trial10419.2.3-689-gf90f4eaefalseOK58105821mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest41aca810-eaf7-41b2-b651-7211a4bd4b03up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}57915796mirror.0trial1042026-02-22 22:06:35cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:06:41.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:06:41.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:06:41.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:35' 2026-02-22T22:06:41.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:06:41.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:06:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:06:41.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror/ns1 test 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror/ns1-test.export 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror/ns1-test.export 2026-02-22T22:06:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:41.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.export 2026-02-22T22:06:41.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror/ns1-test.export 2026-02-22T22:06:41.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror/ns1-test.export 2026-02-22T22:06:41.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test.export 2026-02-22T22:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test.export 2026-02-22T22:06:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/ns1/test /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.export 2026-02-22T22:06:41.595 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:41.600 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/ns1/test /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test.export 2026-02-22T22:06:41.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:41.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test.export 2026-02-22T22:06:41.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test.export 2026-02-22T22:06:41.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:41.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror/ns2 test 2026-02-22T22:06:41.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:41.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:41.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:06:41.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror/ns2-test.export 2026-02-22T22:06:41.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror/ns2-test.export 2026-02-22T22:06:41.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:41.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.export 2026-02-22T22:06:41.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror/ns2-test.export 2026-02-22T22:06:41.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror/ns2-test.export 2026-02-22T22:06:41.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:41.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror_ns2-test.export 2026-02-22T22:06:41.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns2-test.export 2026-02-22T22:06:41.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/ns2/test /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.export 2026-02-22T22:06:41.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:41.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/ns2/test /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns2-test.export 2026-02-22T22:06:41.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:41.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns2-test.export 2026-02-22T22:06:42.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns2-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns2-test.export 2026-02-22T22:06:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - disable mirroring / delete image' 2026-02-22T22:06:42.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:42.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:42 - disable mirroring / delete image 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:42 - disable mirroring / delete image 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror/ns1 test 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:42.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror/ns1 test 2026-02-22T22:06:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test 2026-02-22T22:06:42.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test 2026-02-22T22:06:42.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ disable_mirror cluster2 mirror/ns2 test 2026-02-22T22:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 mirror image disable mirror/ns2/test 2026-02-22T22:06:42.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Mirroring disabled 2026-02-22T22:06:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror/ns1 test deleted 2026-02-22T22:06:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-22T22:06:42.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:42.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:06:42.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:42.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:42.296 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror/ns1 test deleted '' 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror/ns1 test 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:42.297 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/ns1/test 2026-02-22T22:06:42.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:42.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:42.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror/ns2 test deleted 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:42.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-22T22:06:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns2 2026-02-22T22:06:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-22T22:06:42.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:42.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:42.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:06:42.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:42.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror/ns2 test deleted '' 2026-02-22T22:06:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:42.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror/ns2 test 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns2 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/ns2/test 2026-02-22T22:06:42.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:42.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:42.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:42.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:42.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:06:42.473 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror/ns2 test 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror/ns2 test 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns2 2026-02-22T22:06:42.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:42.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns2/test 2026-02-22T22:06:42.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/ns2/test 2026-02-22T22:06:42.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:06:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - data pool' 2026-02-22T22:06:42.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:42.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:42 - data pool 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:42 - data pool 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ dp_image=test_data_pool 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror/ns1 test_data_pool journal 128 --data-pool mirror_parent 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror/ns1 test_data_pool 128 --data-pool mirror_parent 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.558 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:06:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 128 ']' 2026-02-22T22:06:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=128 2026-02-22T22:06:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:42.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling --data-pool mirror_parent mirror/ns1/test_data_pool 2026-02-22T22:06:42.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T22:06:42.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:06:42.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:06:42.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:06:42.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_data_pool cluster2 mirror/ns1 test_data_pool 2026-02-22T22:06:42.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:42.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:42.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:06:42.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 info mirror/ns1/test_data_pool 2026-02-22T22:06:42.590 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T22:06:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ data_pool=mirror_parent 2026-02-22T22:06:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test mirror_parent = mirror_parent 2026-02-22T22:06:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror/ns1 test_data_pool 2026-02-22T22:06:42.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:42.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror/ns1 test_data_pool started 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:43.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:43.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:43.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:43.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:43.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:43.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:43.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:06:43.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:43.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:43.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:43.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:43.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:43.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:43.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:43.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:43.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:43.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:43.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:43.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:06:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:43.741 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:43.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:43.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:43.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:06:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:06:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:06:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:43.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:43.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:43.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:43.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:06:44.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:44.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:44.841 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:44.842 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:44.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:44.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:44.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:44.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:44.896 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:44.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:44.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:06:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:44.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:44.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:44.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:44.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:44.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:44.953 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:06:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:44.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:44.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:06:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:44.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:06:44.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:45.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:06:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:45.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:45.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:45.008 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:06:45.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:06:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:06:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/ns1/test_data_pool'] 2026-02-22T22:06:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:06:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:06:45.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:45.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T22:06:47.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:47.060 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:47.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:47.064 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:47.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:47.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:47.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:47.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/ns1/test_data_pool 2026-02-22T22:06:47.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:06:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:06:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:47.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:06:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:06:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:06:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_data_pool cluster1 mirror/ns1 test_data_pool 2026-02-22T22:06:47.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:47.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:47.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:06:47.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 info mirror/ns1/test_data_pool 2026-02-22T22:06:47.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ awk '$1 == "data_pool:" {print $2}' 2026-02-22T22:06:47.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ data_pool=mirror_parent 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test mirror_parent = mirror_parent 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror/ns1 test_data_pool 100 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:06:47.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/ns1/test_data_pool --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:06:47.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:06:47.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:06:47.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 9091.09 bytes/sec: 36 MiB/s 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror/ns1 test_data_pool 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror/ns1 test_data_pool 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:47.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:06:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:06:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:06:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:06:47.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror/ns1 test_data_pool 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_data_pool ']' 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/ns1/test_data_pool' 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/ns1/test_data_pool 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:47.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:47.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/ns1/test_data_pool 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:47.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:47.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:47.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/ns1/test_data_pool 2026-02-22T22:06:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror/ns1 test_data_pool 2026-02-22T22:06:47.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:06:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test_data_pool '' 2026-02-22T22:06:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:06:47.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:06:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test_data_pool.status 2026-02-22T22:06:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test_data_pool.status 2026-02-22T22:06:47.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T22:06:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test_data_pool 2026-02-22T22:06:47.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T22:06:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:47.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T22:06:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror/ns1 test_data_pool 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror/ns1 test_data_pool '..*' 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror/ns1 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_data_pool 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:06:47.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror/ns1-test_data_pool.status 2026-02-22T22:06:47.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror/ns1-test_data_pool.status 2026-02-22T22:06:47.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:47.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T22:06:47.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/ns1/test_data_pool 2026-02-22T22:06:47.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T22:06:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:06:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:06:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:06:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:47.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:06:47.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.status 2026-02-22T22:06:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:06:47.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror/ns1 test_data_pool 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying primary_position 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:47.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:47.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror/ns1 test_data_pool up+replaying primary_position '' 2026-02-22T22:06:48.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:48.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/ns1/test_data_pool 2026-02-22T22:06:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_data_pool: 2026-02-22T22:06:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 2933d998-9f42-4719-8960-6837f8a9c6d9 2026-02-22T22:06:48.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:45 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:06:48.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:06:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:06:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:48.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:48.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:06:48.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:48.570 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:06:48.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:06:48.571 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:06:48.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:45' 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror/ns1 test_data_pool up+replaying '2026-02-22 22:06:45' 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:06:45' 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:48.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T22:06:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING157915796mirror.0trial10419.2.3-689-gf90f4eaetrueOK57985801mirror.1trial10419.2.3-689-gf90f4eaefalseOK58035808mirror.2trial10419.2.3-689-gf90f4eaefalseOK58105821mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_data_pool2933d998-9f42-4719-8960-6837f8a9c6d9up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}57915796mirror.0trial1042026-02-22 22:06:45cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:06:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:06:48.607 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/last_update' 2026-02-22T22:06:48.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:45' 2026-02-22T22:06:48.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_data_pool'\'']/state' 2026-02-22T22:06:48.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:06:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:06:48.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:48.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror/ns1 test_data_pool 2026-02-22T22:06:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:06:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror/ns1-test_data_pool.export 2026-02-22T22:06:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror/ns1-test_data_pool.export 2026-02-22T22:06:48.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.export 2026-02-22T22:06:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror/ns1-test_data_pool.export 2026-02-22T22:06:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror/ns1-test_data_pool.export 2026-02-22T22:06:48.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T22:06:48.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T22:06:48.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/ns1/test_data_pool /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.export 2026-02-22T22:06:48.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:48.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/ns1/test_data_pool /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T22:06:48.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:48.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T22:06:48.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror_ns1-test_data_pool.export /tmp/tmp.S4fo03ccUK/cluster1-mirror_ns1-test_data_pool.export 2026-02-22T22:06:48.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror/ns1 test_data_pool 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror/ns1 test_data_pool 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:06:48.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror/ns1 2026-02-22T22:06:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_data_pool 2026-02-22T22:06:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/ns1/test_data_pool 2026-02-22T22:06:48.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/ns1/test_data_pool 2026-02-22T22:06:48.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:06:48.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: simple image resync' 2026-02-22T22:06:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:48.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:48 TEST: simple image resync 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:48 TEST: simple image resync 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id_var_name=image_id 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:48.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'image_id='\''16a45866ec7e'\''' 2026-02-22T22:06:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ image_id=16a45866ec7e 2026-02-22T22:06:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'test -n "$image_id"' 2026-02-22T22:06:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -n 16a45866ec7e 2026-02-22T22:06:48.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T22:06:49.024 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Flagged image for resync from primary 2026-02-22T22:06:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test deleted 16a45866ec7e 2026-02-22T22:06:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:49.027 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:06:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a45866ec7e 2026-02-22T22:06:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a45866ec7e 2026-02-22T22:06:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:49.028 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:49.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 16a45866ec7e 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a45866ec7e 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:49.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:49.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:49.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:49.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:49.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:49.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:06:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:49.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T22:06:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:49.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:49.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:49.302 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:49.304 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:06:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:49.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present '' 2026-02-22T22:06:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:50.305 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:50.306 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=16a48a66ca1b 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a48a66ca1b 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 16a48a66ca1b 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:50.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:06:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:50.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:06:51.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:51.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:51.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:06:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:51.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:51.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:06:51.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:06:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:06:51.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:51.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:06:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:06:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:06:51.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:51.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:51.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T22:06:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:52.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:52.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:52.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:52.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:52.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:06:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T22:06:52.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 102b7139-538b-4ca7-96f8-0427c62915f1 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:51 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:06:52.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:06:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:06:52.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:06:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:06:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:52.439 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:06:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:52.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:06:52.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:06:52.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:06:52.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:06:52.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:51' 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:06:51' 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:06:51' 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:52.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:06:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING157915796mirror.0trial10419.2.3-689-gf90f4eaetrueOK57985801mirror.1trial10419.2.3-689-gf90f4eaefalseOK58035808mirror.2trial10419.2.3-689-gf90f4eaefalseOK58105821mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest102b7139-538b-4ca7-96f8-0427c62915f1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}57915796mirror.0trial1042026-02-22 22:06:51cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:06:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:06:52.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:06:52.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:51' 2026-02-22T22:06:52.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:06:52.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:06:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:06:52.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:52.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T22:06:52.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T22:06:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T22:06:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T22:06:52.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:52.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:52.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:52.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T22:06:52.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:52.572 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:52.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:52.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:52.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:52.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:52.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T22:06:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: image resync while replayer is stopped' 2026-02-22T22:06:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:52.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:52 TEST: image resync while replayer is stopped 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:52 TEST: image resync while replayer is stopped 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror stop mirror/test 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:52.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:52.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror/test 2026-02-22T22:06:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:52.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:52.750 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:52.751 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:52.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:52.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror/test 2026-02-22T22:06:52.810 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test stopped 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:52.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:52.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test stopped 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:53.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:53.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:06:53.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:53.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:53.819 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:53.820 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:53.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:06:53.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:53.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:06:53.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:06:53.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:06:53.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id_var_name=image_id 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:53.882 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:53.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:53.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:53.883 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'image_id='\''16a48a66ca1b'\''' 2026-02-22T22:06:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ image_id=16a48a66ca1b 2026-02-22T22:06:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'test -n "$image_id"' 2026-02-22T22:06:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -n 16a48a66ca1b 2026-02-22T22:06:53.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T22:06:53.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Flagged image for resync from primary 2026-02-22T22:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T22:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:53.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:53.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:53.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:53.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:53.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:53.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:53.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:53.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:53.957 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T22:06:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test deleted 16a48a66ca1b 2026-02-22T22:06:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:54.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:06:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a48a66ca1b 2026-02-22T22:06:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a48a66ca1b 2026-02-22T22:06:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:54.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:54.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 16a48a66ca1b 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a48a66ca1b 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:54.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:54.126 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image test: (2) No such file or directory 2026-02-22T22:06:54.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror start mirror/test 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:06:54.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:06:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:54.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:54.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/test 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:06:54.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:54.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:54.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:54.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:54.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/test 2026-02-22T22:06:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T22:06:54.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:54.191 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id=16a4c3a3becb 2026-02-22T22:06:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:06:54.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:06:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present 16a4c3a3becb 2026-02-22T22:06:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:06:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a4c3a3becb 2026-02-22T22:06:54.319 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:06:54.320 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:06:54.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=16a4c3a3becb 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a4c3a3becb 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 16a4c3a3becb -o 16a4c3a3becb = 16a4c3a3becb 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:54.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:54.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:06:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:06:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:06:55.354 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:06:55.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:06:55.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:55.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:55.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:55.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:55.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:55.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:55.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:06:55.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:06:55.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:06:55.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:06:55.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:06:55.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:06:55.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:06:55.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:55.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:55.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:55.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:55.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:06:55.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:06:56.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:56.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:06:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T22:06:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 102b7139-538b-4ca7-96f8-0427c62915f1 2026-02-22T22:06:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:06:56.456 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}} 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:06:55 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:06:56.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:06:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:06:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:06:56.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}} 2026-02-22T22:06:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:06:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:06:56.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:06:56.460 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:06:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:06:56.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:06:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:55' 2026-02-22T22:06:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:06:55' 2026-02-22T22:06:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:56.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:06:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:06:55' 2026-02-22T22:06:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:06:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:06:56.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:06:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING157915796mirror.0trial10419.2.3-689-gf90f4eaetrueOK57985801mirror.1trial10419.2.3-689-gf90f4eaefalseOK58035808mirror.2trial10419.2.3-689-gf90f4eaefalseOK58105821mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest102b7139-538b-4ca7-96f8-0427c62915f1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3}}57915796mirror.0trial1042026-02-22 22:06:55cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:06:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:06:56.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:06:56.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:06:55' 2026-02-22T22:06:56.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:06:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:06:56.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:06:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:06:56.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:06:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:56.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T22:06:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:06:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:06:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:06:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T22:06:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T22:06:56.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T22:06:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T22:06:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T22:06:56.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:06:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:56.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:56.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T22:06:56.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:56.592 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:56.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:06:56.686 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:56.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:06:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:06:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: request image resync while daemon is offline' 2026-02-22T22:06:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:06:56.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:06:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:06:56 TEST: request image resync while daemon is offline 2026-02-22T22:06:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:06:56 TEST: request image resync while daemon is offline 2026-02-22T22:06:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster1 2026-02-22T22:06:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:06:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:56.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:06:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:06:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:06:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:56.773 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:0 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:56.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:06:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:06:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:06:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:06:56.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:56.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=87367 2026-02-22T22:06:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 87367 ']' 2026-02-22T22:06:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 87367 2026-02-22T22:06:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:56.786 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:56.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87367 '$2 == pid {print; exit 1}' 2026-02-22T22:06:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 87367 1.0 0.0 755952 64684 ? Ssl 22:06 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:56.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:57.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87367 '$2 == pid {print; exit 1}' 2026-02-22T22:06:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:57.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87367 '$2 == pid {print; exit 1}' 2026-02-22T22:06:57.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:06:57.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:57.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:57.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:57.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:57.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:57.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:06:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:57.850 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:06:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:06:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:06:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:06:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:06:57.852 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:06:57.853 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:06:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:57.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:06:57.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:1 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:06:57.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:06:57.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:06:57.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:57.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=87410 2026-02-22T22:06:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 87410 ']' 2026-02-22T22:06:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 87410 2026-02-22T22:06:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:57.877 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:57.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87410 '$2 == pid {print; exit 1}' 2026-02-22T22:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 87410 0.2 0.0 670916 43728 ? Ssl 22:06 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:57.887 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:58.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87410 '$2 == pid {print; exit 1}' 2026-02-22T22:06:58.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:58.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:58.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87410 '$2 == pid {print; exit 1}' 2026-02-22T22:06:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:06:58.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:58.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:58.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:58.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:58.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:58.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:58.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:06:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:58.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:06:58.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:06:58.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:06:58.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:58.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:06:58.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:06:58.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:58.947 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:06:58.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:06:58.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T22:06:58.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:06:58.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:2 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:58.950 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:06:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:06:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:06:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:06:58.951 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:06:58.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:06:58.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:06:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=87471 2026-02-22T22:06:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 87471 ']' 2026-02-22T22:06:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 87471 2026-02-22T22:06:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:58.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:06:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:58.963 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87471 '$2 == pid {print; exit 1}' 2026-02-22T22:06:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 87471 0.2 0.0 670916 43952 ? Ssl 22:06 0:00 rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:06:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:06:58.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:59.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87471 '$2 == pid {print; exit 1}' 2026-02-22T22:06:59.984 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:06:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:06:59.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87471 '$2 == pid {print; exit 1}' 2026-02-22T22:06:59.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:06:59.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:06:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:06:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:06:59.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:07:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:00.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:07:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:07:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:00.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:07:00.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:07:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:00.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:07:00.022 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:07:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:07:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:07:00.023 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:07:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:07:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:07:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T22:07:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:07:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:07:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:07:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:07:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:3 2026-02-22T22:07:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:07:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:07:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:07:00.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:07:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:07:00.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:07:00.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:07:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=87523 2026-02-22T22:07:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 87523 ']' 2026-02-22T22:07:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 87523 2026-02-22T22:07:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:07:00.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:07:00.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87523 '$2 == pid {print; exit 1}' 2026-02-22T22:07:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 87523 0.2 0.0 670912 44916 ? Ssl 22:06 0:00 rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:07:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:07:00.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:07:01.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87523 '$2 == pid {print; exit 1}' 2026-02-22T22:07:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:07:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:07:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=87523 '$2 == pid {print; exit 1}' 2026-02-22T22:07:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:07:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:07:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:07:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:07:01.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:01.082 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:07:01.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:01.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:01.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:01.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:07:01.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:01.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:07:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:07:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:01.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:07:01.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:07:01.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:07:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ request_resync_image cluster1 mirror test image_id 2026-02-22T22:07:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id_var_name=image_id 2026-02-22T22:07:01.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:07:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:07:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:07:01.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:01.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'image_id='\''16a4c3a3becb'\''' 2026-02-22T22:07:01.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ image_id=16a4c3a3becb 2026-02-22T22:07:01.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'test -n "$image_id"' 2026-02-22T22:07:01.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -n 16a4c3a3becb 2026-02-22T22:07:01.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/test 2026-02-22T22:07:01.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Flagged image for resync from primary 2026-02-22T22:07:01.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster1 2026-02-22T22:07:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:01.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:0 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:01.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:01.179 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:07:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:1 2026-02-22T22:07:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:01.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:01.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:2 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:01.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:01.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:07:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:01.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:3 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:01.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:07:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:07:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:01.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:07:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test deleted 16a4c3a3becb 2026-02-22T22:07:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:01.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:07:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a4c3a3becb 2026-02-22T22:07:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a4c3a3becb 2026-02-22T22:07:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:01.254 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 16a4c3a3becb 2026-02-22T22:07:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:01.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a4c3a3becb 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:07:01.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=16a4c3a3becb 2026-02-22T22:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a4c3a3becb 2026-02-22T22:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 16a4c3a3becb -o 16a4c3a3becb = 16a4c3a3becb 2026-02-22T22:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:01.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 16a4c3a3becb 2026-02-22T22:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a4c3a3becb 2026-02-22T22:07:02.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:07:02.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=16a4c3a3becb 2026-02-22T22:07:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a4c3a3becb 2026-02-22T22:07:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 16a4c3a3becb -o 16a4c3a3becb = 16a4c3a3becb 2026-02-22T22:07:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:07:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:02.416 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:07:04.418 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 16a4c3a3becb 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a4c3a3becb 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:04.419 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:07:04.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:07:04.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:04.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=16a4c3a3becb 2026-02-22T22:07:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16a4c3a3becb 2026-02-22T22:07:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 16a4c3a3becb -o 16a4c3a3becb = 16a4c3a3becb 2026-02-22T22:07:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:07:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:04.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:07:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test deleted 16a4c3a3becb 2026-02-22T22:07:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=16a4c3a3becb 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:07:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:07:08.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=1747d8b685a4 2026-02-22T22:07:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 1747d8b685a4 2026-02-22T22:07:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 16a4c3a3becb -o 16a4c3a3becb = 1747d8b685a4 2026-02-22T22:07:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:07:08.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror test present 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:07:08.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:07:08.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id=1747d8b685a4 2026-02-22T22:07:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:08.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror test present 1747d8b685a4 2026-02-22T22:07:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:08.610 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=1747d8b685a4 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror test 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/test 2026-02-22T22:07:08.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=1747d8b685a4 2026-02-22T22:07:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 1747d8b685a4 2026-02-22T22:07:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 1747d8b685a4 -o 1747d8b685a4 = 1747d8b685a4 2026-02-22T22:07:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:07:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:08.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test started 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:08.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:09.640 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test started 2026-02-22T22:07:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:07:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:09.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:09.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test 2026-02-22T22:07:09.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:09.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:09.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test 2026-02-22T22:07:09.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:09.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:09.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:09.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:09.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test 2026-02-22T22:07:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test'] 2026-02-22T22:07:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:09.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test 2026-02-22T22:07:09.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:09.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:09.704 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:09.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:09.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:09.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:09.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:09.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test 2026-02-22T22:07:09.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:07:09.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:07:09.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:09.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:07:09.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:07:09.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test up+replaying primary_position 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:09.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:07:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:07:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:09.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:10.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test up+replaying primary_position '' 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:10.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test: 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 102b7139-538b-4ca7-96f8-0427c62915f1 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":4}} 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:07:07 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:07:10.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:07:10.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:07:10.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:07:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":4}} 2026-02-22T22:07:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:07:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:07:10.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:07:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:07:10.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:07:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:07:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:07:10.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:07' 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test up+replaying '2026-02-22 22:07:07' 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:07:07' 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:10.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:07:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest102b7139-538b-4ca7-96f8-0427c62915f1up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":3},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":4}}59545959mirror.1trial1042026-02-22 22:07:07cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:07:10.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:07:10.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/last_update' 2026-02-22T22:07:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:07' 2026-02-22T22:07:10.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test'\'']/state' 2026-02-22T22:07:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:07:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:07:10.835 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror test 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:07:10.836 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-test.export 2026-02-22T22:07:10.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-test.export 2026-02-22T22:07:10.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:10.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T22:07:10.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-test.export 2026-02-22T22:07:10.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-test.export 2026-02-22T22:07:10.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:07:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:07:10.840 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/test /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export 2026-02-22T22:07:10.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:10.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/test /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:07:11.016 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:11.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:07:11.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-test.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-test.export 2026-02-22T22:07:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test 2026-02-22T22:07:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:07:11.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test 2026-02-22T22:07:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test 2026-02-22T22:07:11.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test 2026-02-22T22:07:11.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test 2026-02-22T22:07:11.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:07:11.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: client disconnect' 2026-02-22T22:07:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:11.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:11 TEST: client disconnect 2026-02-22T22:07:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:11 TEST: client disconnect 2026-02-22T22:07:11.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image=laggy 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror laggy 128 --journal-object-size 64K 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 128 ']' 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=128 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:11.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling --journal-object-size 64K mirror/laggy 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror laggy 10 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=10 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:07:11.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/laggy --io-type write --io-size 4096 --io-threads 1 --io-total 40960 --io-pattern rand 2026-02-22T22:07:11.264 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 40960 pattern random 2026-02-22T22:07:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:07:11.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 10 ops/sec: 3333.4 bytes/sec: 13 MiB/s 2026-02-22T22:07:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - replay stopped after disconnect' 2026-02-22T22:07:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:11.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:11 - replay stopped after disconnect 2026-02-22T22:07:11.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:11 - replay stopped after disconnect 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror laggy 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:11.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:07:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:12.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:12.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:12.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:12.278 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:12.281 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:12.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:12.283 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:12.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:12.292 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:12.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:12.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:12.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:12.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:12.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:12.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:12.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:12.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:12.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:12.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/laggy 2026-02-22T22:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:07:12.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:07:12.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:07:12.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:07:12.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:07:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:12.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/laggy 2026-02-22T22:07:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:12.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:12.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/laggy 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:07:12.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:12.450 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:07:12.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:07:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:12.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:12.459 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/laggy 2026-02-22T22:07:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:12.500 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:12.501 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:07:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:13.502 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:13.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:13.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:13.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:13.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:13.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:13.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:13.518 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:13.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:13.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:13.560 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:13.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:13.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:13.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:13.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/laggy 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:07:13.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:07:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:07:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:07:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:07:13.627 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/laggy 2026-02-22T22:07:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:13.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/laggy 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:13.671 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:07:13.672 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:07:13.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:07:13.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:13.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:13.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:07:13.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/laggy 2026-02-22T22:07:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:13.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:13.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T22:07:15.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:15.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:15.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:15.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:15.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:15.732 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:15.742 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:15.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:15.787 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:15.788 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:15.789 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:15.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:15.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:07:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:07:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:15.844 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:07:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:07:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:07:15.846 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:15.847 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:07:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:07:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:15.848 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:16.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:16.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:16.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:16.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:16.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:16.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:16.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:16.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:16.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:16.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:16.108 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:16.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:16.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:16.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:16.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=11], [object_number=2, tag_tid=1, entry_tid=10]]], state=connected] 2026-02-22T22:07:16.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:07:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:16.196 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:16.197 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:16.198 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=11], [object_number=2, tag_tid=1, entry_tid=10]]], state=connected] 2026-02-22T22:07:16.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:07:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:07:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' '' 2026-02-22T22:07:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:07:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' = '' 2026-02-22T22:07:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:07:16.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:07:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:07:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:16.229 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=2 2026-02-22T22:07:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:07:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:07:16.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:16.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=2 2026-02-22T22:07:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:07:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:07:16.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=3 2026-02-22T22:07:16.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:07:16.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:07:16.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=1 2026-02-22T22:07:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 2 = 2 -a 3 -le 1 2026-02-22T22:07:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:16.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:16.437 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:16.438 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:16.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:16.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:16.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:16.451 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:16.452 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:16.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:16.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:16.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:16.498 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:16.499 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:16.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:16.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:16.554 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:16.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:16.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:16.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:16.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:16.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:16.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:16.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:16.589 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:16.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:16.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:16.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:16.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:16.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:16.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:07:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ disconnect_image cluster2 mirror laggy 2026-02-22T22:07:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:16.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:16.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:16.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 journal client disconnect --image mirror/laggy 2026-02-22T22:07:16.677 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:client 5fac3253-137c-4f27-ad9d-6c8f7a882c9d disconnected 2026-02-22T22:07:16.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:16.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:16.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:16.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:16.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:16.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:16.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:16.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:16.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:16.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:16.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:16.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:07:16.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=disconnected] 2026-02-22T22:07:16.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' 2026-02-22T22:07:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror laggy 2026-02-22T22:07:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:16.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:07:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:16.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:17.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:17.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:17.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:17.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:17.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:17.720 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:17.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:17.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:17.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:17.774 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:17.775 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:17.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:17.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:17.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:17.785 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:17.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:07:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:07:17.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:07:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror laggy up+error disconnected 2026-02-22T22:07:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:17.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=disconnected 2026-02-22T22:07:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:17.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror laggy up+error disconnected '' 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=disconnected 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:18.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/laggy 2026-02-22T22:07:18.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='laggy: 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 08045c8c-6c4d-47b8-a5df-07dd072feff0 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+error 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: disconnected 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:07:16 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+error' 2026-02-22T22:07:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+error 2026-02-22T22:07:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*disconnected' 2026-02-22T22:07:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: disconnected 2026-02-22T22:07:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:07:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:07:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:07:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:07:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:07:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:07:18.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:07:18.874 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:07:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:16' 2026-02-22T22:07:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror laggy up+error '2026-02-22 22:07:16' 2026-02-22T22:07:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:18.875 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:07:16' 2026-02-22T22:07:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:18.876 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:07:18.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKlaggy08045c8c-6c4d-47b8-a5df-07dd072feff0up+errordisconnected59545959mirror.1trial1042026-02-22 22:07:16cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:07:18.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:07:18.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''laggy'\'']/last_update' 2026-02-22T22:07:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:16' 2026-02-22T22:07:18.908 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''laggy'\'']/state' 2026-02-22T22:07:18.909 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+error 2026-02-22T22:07:18.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:07:18.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+error 2026-02-22T22:07:18.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:07:18.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:18.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:18.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - replay started after resync requested' 2026-02-22T22:07:18.911 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:18.912 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:18 - replay started after resync requested 2026-02-22T22:07:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:18 - replay started after resync requested 2026-02-22T22:07:18.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ request_resync_image cluster1 mirror laggy image_id 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id_var_name=image_id 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:18.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:18.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'image_id='\''174714e6855f'\''' 2026-02-22T22:07:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ image_id=174714e6855f 2026-02-22T22:07:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'test -n "$image_id"' 2026-02-22T22:07:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -n 174714e6855f 2026-02-22T22:07:18.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/laggy 2026-02-22T22:07:18.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Flagged image for resync from primary 2026-02-22T22:07:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror laggy deleted 174714e6855f 2026-02-22T22:07:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:18.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:07:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=174714e6855f 2026-02-22T22:07:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 174714e6855f 2026-02-22T22:07:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:18.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy deleted 174714e6855f 2026-02-22T22:07:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:19.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=174714e6855f 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:19.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=174714e6855f 2026-02-22T22:07:19.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 174714e6855f 2026-02-22T22:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 174714e6855f -o 174714e6855f = 174714e6855f 2026-02-22T22:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:19.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy deleted 174714e6855f 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=174714e6855f 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:20.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:20.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:20.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:20.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:20.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=174714e6855f 2026-02-22T22:07:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 174714e6855f 2026-02-22T22:07:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 174714e6855f -o 174714e6855f = 174714e6855f 2026-02-22T22:07:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:07:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:20.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:07:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy deleted 174714e6855f 2026-02-22T22:07:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:22.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=174714e6855f 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:22.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:22.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:07:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:22.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror laggy present 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:22.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:22.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:22.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:22.186 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:07:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:22.187 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:22.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:22.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:22.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:22.288 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:22.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:22.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:23.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:23.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:23.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:23.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:23.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:23.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:23.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:23.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:07:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:25.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:25.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:25.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:25.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:25.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:25.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:07:29.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:29.375 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=1747c8c8b1d5 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 1747c8c8b1d5 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 1747c8c8b1d5 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror laggy 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:29.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:30.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:30.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:30.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:30.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:30.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:30.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:30.414 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:30.420 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:30.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:30.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:30.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:30.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:30.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:30.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:30.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:30.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:30.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:30.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:30.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:30.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:30.477 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:30.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:07:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:07:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:30.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:07:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:07:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:07:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:30.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:30.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:30.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:30.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:30.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:30.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:30.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:30.730 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:30.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:30.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:30.778 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:30.781 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:30.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:30.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:30.784 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:30.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:30.837 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:30.838 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:30.839 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:30.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:30.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.870 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T22:07:30.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:30.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:30.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:30.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:30.871 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:30.872 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:30.873 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:30.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:30.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:30.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:30.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T22:07:30.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=3, entry_tid=3' -a 'object_number=3, tag_tid=3, entry_tid=3' = 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T22:07:30.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:30.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:30.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:30.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:30.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:30.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:30.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T22:07:30.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror laggy 2026-02-22T22:07:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:07:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-laggy.export 2026-02-22T22:07:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-laggy.export 2026-02-22T22:07:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:30.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export 2026-02-22T22:07:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-laggy.export 2026-02-22T22:07:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-laggy.export 2026-02-22T22:07:30.934 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:30.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:30.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:30.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/laggy /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export 2026-02-22T22:07:30.988 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:30.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/laggy /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:31.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:31.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:31.128 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:31.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:31.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - disconnected after max_concurrent_object_sets reached' 2026-02-22T22:07:31.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:31.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:31.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:31 - disconnected after max_concurrent_object_sets reached 2026-02-22T22:07:31.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:31 - disconnected after max_concurrent_object_sets reached 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror stop mirror/laggy 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:31.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:31.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror stop mirror/laggy 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:31.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:31.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:31.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror stop mirror/laggy 2026-02-22T22:07:31.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:31.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror stop : stop rbd mirror 2026-02-22T22:07:31.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror stop mirror/laggy 2026-02-22T22:07:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:31.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:31.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:31.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:31.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:31.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:31.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:31.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:31.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror stop mirror/laggy 2026-02-22T22:07:31.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:31.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror laggy 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:31.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:07:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:32.272 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:32.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:32.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:32.276 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:32.277 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:32.289 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:32.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:32.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:32.334 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:32.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:32.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:32.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:32.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:32.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:32.344 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:32.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:32.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:07:32.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:07:32.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:32.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:07:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:32.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:32.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:32.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:32.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:07:32.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:07:32.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:32.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:32.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=connected] 2026-02-22T22:07:32.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=3, entry_tid=3' 2026-02-22T22:07:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_image_meta cluster2 mirror laggy conf_rbd_journal_max_concurrent_object_sets 1 2026-02-22T22:07:32.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=conf_rbd_journal_max_concurrent_object_sets 2026-02-22T22:07:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=1 2026-02-22T22:07:32.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 image-meta set mirror/laggy conf_rbd_journal_max_concurrent_object_sets 1 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror laggy 20 16384 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=20 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=16384 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16384 2026-02-22T22:07:32.462 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/laggy --io-type write --io-size 16384 --io-threads 1 --io-total 327680 --io-pattern rand 2026-02-22T22:07:32.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 16384 io_threads 1 bytes 327680 pattern random 2026-02-22T22:07:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:07:32.503 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 20 ops/sec: 2222.27 bytes/sec: 35 MiB/s 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror laggy 20 16384 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=20 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=16384 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 16384 2026-02-22T22:07:32.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/laggy --io-type write --io-size 16384 --io-threads 1 --io-total 327680 --io-pattern rand 2026-02-22T22:07:32.541 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 16384 io_threads 1 bytes 327680 pattern random 2026-02-22T22:07:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:07:32.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 20 ops/sec: 2222.27 bytes/sec: 35 MiB/s 2026-02-22T22:07:32.555 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:32.556 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:32.557 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 1 2026-02-22T22:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=6, entry_tid=41], [object_number=12, tag_tid=6, entry_tid=40], [object_number=27, tag_tid=6, entry_tid=39], [object_number=14, tag_tid=6, entry_tid=38]]], state=connected] 2026-02-22T22:07:32.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=3, entry_tid=3], [object_number=2, tag_tid=3, entry_tid=2], [object_number=1, tag_tid=3, entry_tid=1], [object_number=0, tag_tid=3, entry_tid=0]]], state=disconnected] 2026-02-22T22:07:32.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' 2026-02-22T22:07:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_image_meta cluster2 mirror laggy conf_rbd_journal_max_concurrent_object_sets 0 2026-02-22T22:07:32.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=conf_rbd_journal_max_concurrent_object_sets 2026-02-22T22:07:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=0 2026-02-22T22:07:32.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 image-meta set mirror/laggy conf_rbd_journal_max_concurrent_object_sets 0 2026-02-22T22:07:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - replay is still stopped (disconnected) after restart' 2026-02-22T22:07:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:32.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:32 - replay is still stopped '(disconnected)' after restart 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:32 - replay is still stopped (disconnected) after restart 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror start mirror/laggy 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:32.628 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:32.631 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror start mirror/laggy 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:32.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:32.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:32.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:32.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:32.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:32.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:32.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror start mirror/laggy 2026-02-22T22:07:32.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror start : start rbd mirror 2026-02-22T22:07:32.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror start mirror/laggy 2026-02-22T22:07:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:32.687 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:32.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:32.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:32.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:32.697 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:32.698 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror start mirror/laggy 2026-02-22T22:07:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror laggy 2026-02-22T22:07:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:32.744 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:32.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:07:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:33.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:33.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:33.752 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:33.753 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:33.754 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:33.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:33.802 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:33.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:33.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:33.807 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:33.808 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:33.809 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:33.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:33.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:33.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:07:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:07:33.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror laggy up+error disconnected 2026-02-22T22:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:33.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=disconnected 2026-02-22T22:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:33.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:34.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror laggy up+error disconnected '' 2026-02-22T22:07:34.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:34.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:34.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:34.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:34.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=disconnected 2026-02-22T22:07:34.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:34.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:34.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:34.867 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/laggy 2026-02-22T22:07:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='laggy: 2026-02-22T22:07:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 08045c8c-6c4d-47b8-a5df-07dd072feff0 2026-02-22T22:07:34.898 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+error 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: disconnected 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:07:32 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:07:34.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+error' 2026-02-22T22:07:34.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+error 2026-02-22T22:07:34.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*disconnected' 2026-02-22T22:07:34.900 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: disconnected 2026-02-22T22:07:34.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:07:34.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:07:34.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:07:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:07:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:07:34.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:07:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:07:34.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:07:34.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:32' 2026-02-22T22:07:34.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror laggy up+error '2026-02-22 22:07:32' 2026-02-22T22:07:34.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:34.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:34.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:07:32' 2026-02-22T22:07:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:34.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKlaggy08045c8c-6c4d-47b8-a5df-07dd072feff0up+errordisconnected59545959mirror.1trial1042026-02-22 22:07:32cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:07:34.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''laggy'\'']/last_update' 2026-02-22T22:07:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:32' 2026-02-22T22:07:34.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''laggy'\'']/state' 2026-02-22T22:07:34.938 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+error 2026-02-22T22:07:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:07:34.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+error 2026-02-22T22:07:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:07:34.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - replay started after resync requested' 2026-02-22T22:07:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:34.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:34 - replay started after resync requested 2026-02-22T22:07:34.942 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:34 - replay started after resync requested 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ request_resync_image cluster1 mirror laggy image_id 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id_var_name=image_id 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:34.943 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:34.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'image_id='\''1747c8c8b1d5'\''' 2026-02-22T22:07:34.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ image_id=1747c8c8b1d5 2026-02-22T22:07:34.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'test -n "$image_id"' 2026-02-22T22:07:34.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -n 1747c8c8b1d5 2026-02-22T22:07:34.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/laggy 2026-02-22T22:07:34.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Flagged image for resync from primary 2026-02-22T22:07:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror laggy deleted 1747c8c8b1d5 2026-02-22T22:07:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:35.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:07:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=1747c8c8b1d5 2026-02-22T22:07:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 1747c8c8b1d5 2026-02-22T22:07:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:35.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:35.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy deleted 1747c8c8b1d5 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=1747c8c8b1d5 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:35.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:35.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=1747c8c8b1d5 2026-02-22T22:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 1747c8c8b1d5 2026-02-22T22:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z 1747c8c8b1d5 -o 1747c8c8b1d5 = 1747c8c8b1d5 2026-02-22T22:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:35.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = present 2026-02-22T22:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:35.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:36.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy deleted 1747c8c8b1d5 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=1747c8c8b1d5 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:36.353 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:36.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror laggy present 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:36.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:36.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:36.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:07:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:36.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:36.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:36.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:36.484 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:36.504 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:36.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:37.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:37.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:37.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:37.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:37.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:37.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:07:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:39.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:39.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:39.559 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:39.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:39.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:07:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:43.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:43.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=17475111cfd2 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 17475111cfd2 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 17475111cfd2 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror laggy 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:43.596 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:43.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:44.597 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:44.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:44.599 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:44.602 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:44.603 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:44.604 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:44.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:44.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:44.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:44.614 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:44.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:44.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:44.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:44.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:44.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:44.669 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:44.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:07:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:07:44.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:44.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:07:44.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:44.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:07:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:07:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:44.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:44.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:44.918 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:44.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:44.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:44.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:44.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:44.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:44.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:44.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:44.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:44.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:44.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:44.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:44.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:44.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:44.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:44.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:44.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:44.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:44.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:45.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:45.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:45.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:45.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:45.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.068 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.069 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.070 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:45.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:45.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:45.071 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:45.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:45.072 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=27, tag_tid=8, entry_tid=3' -a 'object_number=27, tag_tid=8, entry_tid=3' = 'object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:45.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:45.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.107 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.109 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:45.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:45.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:45.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror laggy 2026-02-22T22:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:07:45.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-laggy.export 2026-02-22T22:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-laggy.export 2026-02-22T22:07:45.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export 2026-02-22T22:07:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-laggy.export 2026-02-22T22:07:45.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-laggy.export 2026-02-22T22:07:45.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:45.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/laggy /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export 2026-02-22T22:07:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:45.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/laggy /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:45.307 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:45.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog ' - rbd_mirroring_resync_after_disconnect config option' 2026-02-22T22:07:45.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:45.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:45 - rbd_mirroring_resync_after_disconnect config option 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:45 - rbd_mirroring_resync_after_disconnect config option 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_image_meta cluster2 mirror laggy conf_rbd_mirroring_resync_after_disconnect true 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=conf_rbd_mirroring_resync_after_disconnect 2026-02-22T22:07:45.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=true 2026-02-22T22:07:45.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 image-meta set mirror/laggy conf_rbd_mirroring_resync_after_disconnect true 2026-02-22T22:07:45.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:45.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:45.445 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:45.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:45.646 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:45.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:45.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:45.652 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:45.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:45.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:45.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:45.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:45.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:45.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:45.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:45.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:45.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:45.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:45.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:45.713 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:45.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:45.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:45.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:45.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:45.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:45.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:45.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:45.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=25, tag_tid=9, entry_tid=1' 2026-02-22T22:07:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:45.794 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:45.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:45.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:45.822 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:45.823 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:45.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=25, tag_tid=9, entry_tid=1' -a 'object_number=25, tag_tid=9, entry_tid=1' = 'object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=27, tag_tid=8, entry_tid=3' '!=' '' 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=27, tag_tid=8, entry_tid=3' = '' 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=25, tag_tid=9, entry_tid=1' 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:07:45.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=9 2026-02-22T22:07:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:07:45.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:45.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=8 2026-02-22T22:07:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=25, tag_tid=9, entry_tid=1' 2026-02-22T22:07:45.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:07:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:45.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=1 2026-02-22T22:07:45.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:45.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:07:45.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:07:45.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=3 2026-02-22T22:07:45.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 9 = 8 -a 1 -le 3 2026-02-22T22:07:45.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:45.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:45.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:46.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:46.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:46.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:46.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:46.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:46.042 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:46.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:46.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:46.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:46.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:46.106 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:46.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:46.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:46.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:46.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:46.180 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:46.181 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=25, tag_tid=9, entry_tid=1' 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:46.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:46.184 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:46.185 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:46.213 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2], [object_number=25, tag_tid=8, entry_tid=1], [object_number=24, tag_tid=8, entry_tid=0]]], state=connected] 2026-02-22T22:07:46.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=25, tag_tid=9, entry_tid=1' -a 'object_number=25, tag_tid=9, entry_tid=1' = 'object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=27, tag_tid=8, entry_tid=3' '!=' 'object_number=27, tag_tid=8, entry_tid=3' 2026-02-22T22:07:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:46.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.4 2026-02-22T22:07:46.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:46.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:46.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:46.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:46.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:46.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:46.624 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:46.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:46.632 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:46.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:46.673 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:46.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:46.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:46.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:46.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:46.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:46.684 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:46.685 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:46.688 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:46.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:46.737 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:46.765 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:46.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.767 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=25, tag_tid=9, entry_tid=1' 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:46.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:46.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:46.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:46.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=25, tag_tid=9, entry_tid=1], [object_number=24, tag_tid=9, entry_tid=0], [object_number=27, tag_tid=8, entry_tid=3], [object_number=26, tag_tid=8, entry_tid=2]]], state=connected] 2026-02-22T22:07:46.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=25, tag_tid=9, entry_tid=1' 2026-02-22T22:07:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=25, tag_tid=9, entry_tid=1' -a 'object_number=25, tag_tid=9, entry_tid=1' = 'object_number=25, tag_tid=9, entry_tid=1' 2026-02-22T22:07:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:46.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:46.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id=17475111cfd2 2026-02-22T22:07:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ disconnect_image cluster2 mirror laggy 2026-02-22T22:07:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:46.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 journal client disconnect --image mirror/laggy 2026-02-22T22:07:46.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:client 5fac3253-137c-4f27-ad9d-6c8f7a882c9d disconnected 2026-02-22T22:07:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror laggy deleted 17475111cfd2 2026-02-22T22:07:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=deleted 2026-02-22T22:07:46.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=17475111cfd2 2026-02-22T22:07:46.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:46.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 17475111cfd2 2026-02-22T22:07:46.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:46.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:46.960 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy deleted 17475111cfd2 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=deleted 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id=17475111cfd2 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.961 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:46.962 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:46.983 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test deleted = deleted 2026-02-22T22:07:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_present cluster1 mirror laggy present 2026-02-22T22:07:46.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=present 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:46.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:47.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image_id= 2026-02-22T22:07:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:47.009 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.1 2026-02-22T22:07:47.110 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:47.111 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:47.112 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:47.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:47.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:48.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:48.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:48.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:48.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:50.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:50.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:50.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:50.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image laggy: (2) No such file or directory 2026-02-22T22:07:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id= 2026-02-22T22:07:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:07:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = deleted 2026-02-22T22:07:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.1 1 2 4 8 8 8 8 8 8 8 8 16 16 32 32 2026-02-22T22:07:50.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:07:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_present cluster1 mirror laggy present '' 2026-02-22T22:07:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:54.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=present 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id= 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=deleted 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_image_id 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror laggy 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:54.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/laggy 2026-02-22T22:07:54.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_image_id=1747daa0ee54 2026-02-22T22:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 1747daa0ee54 2026-02-22T22:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' -o '' = 1747daa0ee54 2026-02-22T22:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=present 2026-02-22T22:07:54.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test present = present 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror laggy 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:54.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:54.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:55.225 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy started 2026-02-22T22:07:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:07:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:55.226 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:55.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:55.228 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:55.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:55.232 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:55.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:55.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:55.243 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:55.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:55.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:55.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:55.298 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:55.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:55.346 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:07:55.347 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:07:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:55.348 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:07:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:07:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:07:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:55.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:55.351 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:55.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:55.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:55.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:55.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:55.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:55.606 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:55.608 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:55.609 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:55.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:55.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:55.665 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:55.666 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:55.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.667 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:55.668 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:55.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:55.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:55.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2], [object_number=25, tag_tid=10, entry_tid=1], [object_number=24, tag_tid=10, entry_tid=0]]], state=connected] 2026-02-22T22:07:55.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2], [object_number=25, tag_tid=10, entry_tid=1], [object_number=24, tag_tid=10, entry_tid=0]]], state=connected] 2026-02-22T22:07:55.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=27, tag_tid=10, entry_tid=3' 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:55.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:55.904 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:55.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:55.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:55.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:55.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2], [object_number=25, tag_tid=10, entry_tid=1], [object_number=24, tag_tid=10, entry_tid=0]]], state=connected] 2026-02-22T22:07:55.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2], [object_number=25, tag_tid=10, entry_tid=1], [object_number=24, tag_tid=10, entry_tid=0]]], state=connected] 2026-02-22T22:07:55.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=27, tag_tid=10, entry_tid=3' 2026-02-22T22:07:55.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=27, tag_tid=10, entry_tid=3' -a 'object_number=27, tag_tid=10, entry_tid=3' = 'object_number=27, tag_tid=10, entry_tid=3' 2026-02-22T22:07:55.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:55.935 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:55.936 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:55.937 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2], [object_number=25, tag_tid=10, entry_tid=1], [object_number=24, tag_tid=10, entry_tid=0]]], state=connected] 2026-02-22T22:07:55.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2], [object_number=25, tag_tid=10, entry_tid=1], [object_number=24, tag_tid=10, entry_tid=0]]], state=connected] 2026-02-22T22:07:55.966 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:55.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=27, tag_tid=10, entry_tid=3' 2026-02-22T22:07:55.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ compare_images mirror laggy 2026-02-22T22:07:55.967 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local ret=0 2026-02-22T22:07:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster2-mirror-laggy.export 2026-02-22T22:07:55.968 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster2-mirror-laggy.export 2026-02-22T22:07:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:55.969 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local rmt_export=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export 2026-02-22T22:07:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ mkfname cluster1-mirror-laggy.export 2026-02-22T22:07:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo cluster1-mirror-laggy.export 2026-02-22T22:07:55.970 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:55.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loc_export=/tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:55.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:55.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 export mirror/laggy /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export 2026-02-22T22:07:56.048 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:56.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 export mirror/laggy /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:56.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Exporting image: 3% complete... Exporting image: 6% complete... Exporting image: 9% complete... Exporting image: 12% complete... Exporting image: 15% complete... Exporting image: 18% complete... Exporting image: 21% complete... Exporting image: 25% complete... Exporting image: 28% complete... Exporting image: 31% complete... Exporting image: 34% complete... Exporting image: 37% complete... Exporting image: 40% complete... Exporting image: 43% complete... Exporting image: 46% complete... Exporting image: 50% complete... Exporting image: 53% complete... Exporting image: 56% complete... Exporting image: 59% complete... Exporting image: 62% complete... Exporting image: 65% complete... Exporting image: 68% complete... Exporting image: 71% complete... Exporting image: 75% complete... Exporting image: 78% complete... Exporting image: 81% complete... Exporting image: 84% complete... Exporting image: 87% complete... Exporting image: 90% complete... Exporting image: 93% complete... Exporting image: 96% complete... Exporting image: 100% complete...done. 2026-02-22T22:07:56.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmp /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:56.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.export /tmp/tmp.S4fo03ccUK/cluster1-mirror-laggy.export 2026-02-22T22:07:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_image_meta cluster2 mirror laggy conf_rbd_mirroring_resync_after_disconnect false 2026-02-22T22:07:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:56.223 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local key=conf_rbd_mirroring_resync_after_disconnect 2026-02-22T22:07:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=false 2026-02-22T22:07:56.224 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 image-meta set mirror/laggy conf_rbd_mirroring_resync_after_disconnect false 2026-02-22T22:07:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:56.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror laggy 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:07:56.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:07:56.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror laggy 2026-02-22T22:07:56.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n laggy ']' 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/laggy' 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/laggy 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:56.465 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:07:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:07:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:56.466 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:07:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/laggy 2026-02-22T22:07:56.468 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:56.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:56.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:56.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:56.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:56.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:56.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:07:56.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/laggy 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:07:56.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:56.525 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:56.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:56.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/laggy 2026-02-22T22:07:56.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:56.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror laggy 2026-02-22T22:07:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '' 2026-02-22T22:07:56.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:07:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:56.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:56.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:56.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:56.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:56.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:56.611 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=11, entry_tid=1], [object_number=24, tag_tid=11, entry_tid=0], [object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2]]], state=connected] 2026-02-22T22:07:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=25, tag_tid=11, entry_tid=1], [object_number=24, tag_tid=11, entry_tid=0], [object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2]]], state=connected] 2026-02-22T22:07:56.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.613 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=25, tag_tid=11, entry_tid=1' 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:56.615 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:56.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:56.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:56.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:56.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:56.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=11, entry_tid=1], [object_number=24, tag_tid=11, entry_tid=0], [object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2]]], state=connected] 2026-02-22T22:07:56.644 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=25, tag_tid=11, entry_tid=1], [object_number=24, tag_tid=11, entry_tid=0], [object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2]]], state=connected] 2026-02-22T22:07:56.645 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=25, tag_tid=11, entry_tid=1' 2026-02-22T22:07:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=25, tag_tid=11, entry_tid=1' -a 'object_number=25, tag_tid=11, entry_tid=1' = 'object_number=25, tag_tid=11, entry_tid=1' 2026-02-22T22:07:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ disconnect_image cluster2 mirror laggy 2026-02-22T22:07:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:56.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:56.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 journal client disconnect --image mirror/laggy 2026-02-22T22:07:56.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:client 5fac3253-137c-4f27-ad9d-6c8f7a882c9d disconnected 2026-02-22T22:07:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror laggy 2026-02-22T22:07:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:56.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror laggy '..*' 2026-02-22T22:07:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:07:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:07:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=laggy 2026-02-22T22:07:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:07:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-laggy.status 2026-02-22T22:07:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-laggy.status 2026-02-22T22:07:56.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:07:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/laggy 2026-02-22T22:07:56.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 6 2026-02-22T22:07:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 6 2026-02-22T22:07:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:07:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=25, tag_tid=11, entry_tid=1], [object_number=24, tag_tid=11, entry_tid=0], [object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2]]], state=connected] 2026-02-22T22:07:56.707 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=25, tag_tid=11, entry_tid=1], [object_number=24, tag_tid=11, entry_tid=0], [object_number=27, tag_tid=10, entry_tid=3], [object_number=26, tag_tid=10, entry_tid=2]]], state=disconnected] 2026-02-22T22:07:56.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-laggy.status 2026-02-22T22:07:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -z '' 2026-02-22T22:07:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror laggy 2026-02-22T22:07:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:56.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:07:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:07:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:56.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:57.714 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror laggy stopped 2026-02-22T22:07:57.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:57.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:57.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:57.715 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:07:57.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:07:57.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:07:57.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/laggy 2026-02-22T22:07:57.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:07:57.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:07:57.716 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:07:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:07:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:07:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:07:57.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:07:57.718 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:07:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/laggy 2026-02-22T22:07:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:07:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:57.724 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:07:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:07:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:57.725 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:07:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:07:57.726 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:07:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:07:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:07:57.727 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:07:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:07:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:07:57.728 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:57.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:57.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:07:57.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:07:57.729 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:07:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:07:57.731 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/laggy 2026-02-22T22:07:57.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/laggy'] 2026-02-22T22:07:57.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:07:57.769 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/laggy 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:07:57.771 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:07:57.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:07:57.782 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:07:57.783 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/laggy 2026-02-22T22:07:57.826 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:07:57.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:07:57.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:07:57.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:07:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:07:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror laggy up+error disconnected 2026-02-22T22:07:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:57.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=disconnected 2026-02-22T22:07:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:57.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:07:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror laggy up+error disconnected '' 2026-02-22T22:07:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:58.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=disconnected 2026-02-22T22:07:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:58.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/laggy 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='laggy: 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: 08045c8c-6c4d-47b8-a5df-07dd072feff0 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+error 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: disconnected 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:07:56 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+error' 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+error 2026-02-22T22:07:58.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*disconnected' 2026-02-22T22:07:58.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: disconnected 2026-02-22T22:07:58.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:07:58.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:07:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:07:58.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:07:58.863 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:07:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:07:58.864 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:07:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:56' 2026-02-22T22:07:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror laggy up+error '2026-02-22 22:07:56' 2026-02-22T22:07:58.865 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:58.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:58.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:58.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:07:58.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:07:56' 2026-02-22T22:07:58.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:07:58.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:07:58.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:07:58.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKlaggy08045c8c-6c4d-47b8-a5df-07dd072feff0up+errordisconnected59545959mirror.1trial1042026-02-22 22:07:56cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:07:58.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:07:58.897 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''laggy'\'']/last_update' 2026-02-22T22:07:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:07:56' 2026-02-22T22:07:58.899 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''laggy'\'']/state' 2026-02-22T22:07:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+error 2026-02-22T22:07:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:07:58.901 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+error 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror laggy 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:07:58.902 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:07:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror laggy 2026-02-22T22:07:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=laggy 2026-02-22T22:07:58.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/laggy 2026-02-22T22:07:58.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/laggy 2026-02-22T22:07:58.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:07:58.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:07:58.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: split-brain' 2026-02-22T22:07:58.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:07:58.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:07:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:07:58 TEST: split-brain 2026-02-22T22:07:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:07:58 TEST: split-brain 2026-02-22T22:07:58.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ image=split-brain 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror split-brain journal 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror split-brain 2026-02-22T22:07:58.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:07:58.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/split-brain 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:59.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:07:59.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position '' 2026-02-22T22:08:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:00.029 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:00.030 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:00.054 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-22T22:08:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status= 2026-02-22T22:08:00.055 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:00.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:08:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:00.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:08:02.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position '' 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:02.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:02.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+syncing 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: bootstrapping, IMAGE_SYNC/CREATE_SYNC_POINT 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:01 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:02.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:02.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:08:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:02.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:08:06.094 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position '' 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:06.095 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:08:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:08:06.129 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:06.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:03 2026-02-22T22:08:06.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:06.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:06.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:06.130 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:06.131 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:08:06.132 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:08:06.133 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:08:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:08:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:08:06.134 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:08:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:08:06.135 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:08:06.136 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:08:06.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:03' 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-22 22:08:03' 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:08:03' 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:06.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:06.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:08:06.166 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKsplit-brainff5b05a6-0fc0-42b7-91ed-a3266f8d1647up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}59545959mirror.1trial1042026-02-22 22:08:03cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:08:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:08:06.167 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T22:08:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:03' 2026-02-22T22:08:06.168 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T22:08:06.169 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:08:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:08:06.170 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:08:06.171 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror split-brain --force 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force=--force 2026-02-22T22:08:06.173 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/split-brain --force 2026-02-22T22:08:06.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T22:08:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror split-brain 2026-02-22T22:08:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:06.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror split-brain stopped 2026-02-22T22:08:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:08:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:08:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:06.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror split-brain stopped 2026-02-22T22:08:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:08:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:08:07.210 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/split-brain 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:07.211 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:08:07.214 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/split-brain 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:07.215 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:08:07.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:08:07.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:08:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:07.227 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/split-brain 2026-02-22T22:08:07.267 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/split-brain'] 2026-02-22T22:08:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:07.268 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/split-brain 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:07.273 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:07.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:08:07.275 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:08:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:07.282 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/split-brain 2026-02-22T22:08:07.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:08:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:08:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:08:07.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:08:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:08:07.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+stopped 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:07.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+stopped '' '' 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:08.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:08.363 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:06 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T22:08:08.364 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:08:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T22:08:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:08:08.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:08:08.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:08:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:08:08.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:08:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:08:08.368 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:08:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:06' 2026-02-22T22:08:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+stopped '2026-02-22 22:08:06' 2026-02-22T22:08:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:08.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:08:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:08:06' 2026-02-22T22:08:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:08.370 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:08:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKsplit-brainff5b05a6-0fc0-42b7-91ed-a3266f8d1647up+stoppedlocal image is primary59545959mirror.1trial1042026-02-22 22:08:06cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:08:08.401 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:08:08.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T22:08:08.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:06' 2026-02-22T22:08:08.403 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T22:08:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T22:08:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:08:08.405 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T22:08:08.406 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster1 mirror split-brain 10 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=10 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:08:08.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 bench mirror/split-brain --io-type write --io-size 4096 --io-threads 1 --io-total 40960 --io-pattern rand 2026-02-22T22:08:08.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 40960 pattern random 2026-02-22T22:08:08.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:08:08.444 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 10 ops/sec: 3333.4 bytes/sec: 13 MiB/s 2026-02-22T22:08:08.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ demote_image cluster1 mirror split-brain 2026-02-22T22:08:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:08.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image demote mirror/split-brain 2026-02-22T22:08:08.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image demoted to non-primary 2026-02-22T22:08:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+error split-brain 2026-02-22T22:08:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:08.490 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:08:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=split-brain 2026-02-22T22:08:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:08.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:09.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=split-brain 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:09.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:09.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:06 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:09.521 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+error' 2026-02-22T22:08:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:08:09.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:09.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:08:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+error split-brain '' 2026-02-22T22:08:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:11.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:08:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=split-brain 2026-02-22T22:08:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:11.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:11.755 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+error 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: split-brain detected 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:11 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:11.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+error' 2026-02-22T22:08:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+error 2026-02-22T22:08:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*split-brain' 2026-02-22T22:08:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: split-brain detected 2026-02-22T22:08:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:08:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:08:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:08:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:08:11.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:08:11.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:08:11.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:08:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:08:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:11' 2026-02-22T22:08:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+error '2026-02-22 22:08:11' 2026-02-22T22:08:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:11.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+error 2026-02-22T22:08:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:08:11' 2026-02-22T22:08:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:11.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:08:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKsplit-brainff5b05a6-0fc0-42b7-91ed-a3266f8d1647up+errorsplit-brain detected59545959mirror.1trial1042026-02-22 22:08:11cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:08:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:08:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T22:08:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:11' 2026-02-22T22:08:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T22:08:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+error 2026-02-22T22:08:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+error 2026-02-22T22:08:11.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+error 2026-02-22T22:08:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+error 2026-02-22T22:08:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:11.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ request_resync_image cluster1 mirror split-brain image_id 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image_id_var_name=image_id 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_image_id cluster1 mirror split-brain 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=split-brain 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster=cluster1 info mirror/split-brain 2026-02-22T22:08:11.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -ne 's/^.*block_name_prefix: rbd_data\.//p' 2026-02-22T22:08:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'image_id='\''17473f0058b9'\''' 2026-02-22T22:08:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ image_id=17473f0058b9 2026-02-22T22:08:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval 'test -n "$image_id"' 2026-02-22T22:08:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -n 17473f0058b9 2026-02-22T22:08:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image resync mirror/split-brain 2026-02-22T22:08:11.854 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Flagged image for resync from primary 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror split-brain 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:11.856 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:11.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:12.857 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position '' 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:12.858 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+error 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: split-brain detected 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:11 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:12.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:12.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:12.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:12.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:12.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:08:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:12.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:08:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position '' 2026-02-22T22:08:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:14.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:14.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+error 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: split-brain detected 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:11 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:14.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:08:14.926 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:14.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position '' 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:18.927 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:18.954 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: error opening image split-brain: (2) No such file or directory 2026-02-22T22:08:18.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status= 2026-02-22T22:08:18.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:18.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:08:18.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:18.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 8 2026-02-22T22:08:26.958 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror split-brain up+replaying primary_position '' 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:26.959 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/split-brain 2026-02-22T22:08:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='split-brain: 2026-02-22T22:08:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: ff5b05a6-0fc0-42b7-91ed-a3266f8d1647 2026-02-22T22:08:26.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:23 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: down+unknown 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: status not found 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: ' 2026-02-22T22:08:26.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:08:26.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:08:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:08:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:08:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:08:26.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:08:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:08:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:08:26.997 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:08:26.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:08:26.998 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:23' 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror split-brain up+replaying '2026-02-22 22:08:23' 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:08:23' 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:27.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:27.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:08:27.031 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='WARNINGOKWARNING159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKsplit-brainff5b05a6-0fc0-42b7-91ed-a3266f8d1647up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}59545959mirror.1trial1042026-02-22 22:08:23cluster2605c498e-0476-4c03-b186-dac6ca3f245bdown+unknownstatus not found' 2026-02-22T22:08:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:08:27.032 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/last_update' 2026-02-22T22:08:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:23' 2026-02-22T22:08:27.033 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''split-brain'\'']/state' 2026-02-22T22:08:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:08:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:08:27.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:08:27.036 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:08:27.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:27.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:27.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror split-brain 2026-02-22T22:08:27.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror split-brain 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=split-brain 2026-02-22T22:08:27.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/split-brain 2026-02-22T22:08:27.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/split-brain 2026-02-22T22:08:27.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 3% complete... Removing image: 6% complete... Removing image: 9% complete... Removing image: 12% complete... Removing image: 15% complete... Removing image: 18% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 31% complete... Removing image: 34% complete... Removing image: 37% complete... Removing image: 40% complete... Removing image: 43% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 56% complete... Removing image: 59% complete... Removing image: 62% complete... Removing image: 65% complete... Removing image: 68% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 81% complete... Removing image: 84% complete... Removing image: 87% complete... Removing image: 90% complete... Removing image: 93% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2026-02-22T22:08:27.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: check if removed images'\'' OMAP are removed' 2026-02-22T22:08:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:08:27.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:08:27.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:08:27 TEST: check if removed 'images'\''' OMAP are removed 2026-02-22T22:08:27.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:08:27 TEST: check if removed images' OMAP are removed 2026-02-22T22:08:27.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster2 2026-02-22T22:08:27.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:27.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:08:27.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:0 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:0 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:27.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:1 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:1 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster2:1 ']' 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:27.144 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:27.145 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:2 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:2 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:27.163 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster2:2 ']' 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:27.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:08:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster2:3 2026-02-22T22:08:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:08:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:08:27.182 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster2:3 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster2 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster2:3 ']' 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster2 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:27.183 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster2 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:08:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_in_omap cluster1 mirror 2026-02-22T22:08:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring status_global 2026-02-22T22:08:27.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:08:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=status_global 2026-02-22T22:08:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:27.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring status_global 2026-02-22T22:08:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:27.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=status_global 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c status_global 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:27.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T22:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:27.230 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirroring image_ 2026-02-22T22:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_ 2026-02-22T22:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:27.231 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirroring image_ 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_ 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirroring 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_ 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:27.233 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:27.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:08:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T22:08:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:08:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:08:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:27.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:08:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T22:08:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_map 2026-02-22T22:08:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:27.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:08:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:27.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:27.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:08:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 1 = 0 2026-02-22T22:08:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=1 2026-02-22T22:08:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T22:08:27.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:27.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:28.284 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:08:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T22:08:28.285 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T22:08:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T22:08:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:28.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:08:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 1 = 0 2026-02-22T22:08:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=1 2026-02-22T22:08:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 -eq 0 ']' 2026-02-22T22:08:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:28.311 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:08:30.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster1 mirror rbd_mirror_leader image_map 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster1 mirror rbd_mirror_leader 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:30.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster1 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_in_omap cluster2 mirror 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring status_global 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=status_global 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:30.335 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring status_global 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=status_global 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c status_global 2026-02-22T22:08:30.336 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:30.337 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:08:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T22:08:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:08:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:08:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirroring image_ 2026-02-22T22:08:30.356 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirroring 2026-02-22T22:08:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_ 2026-02-22T22:08:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:30.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirroring image_ 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_ 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster2 mirror rbd_mirroring 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirroring 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_ 2026-02-22T22:08:30.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirroring 2026-02-22T22:08:30.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_omap_keys cluster2 mirror rbd_mirror_leader image_map 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local obj_name=rbd_mirror_leader 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local filter=image_map 2026-02-22T22:08:30.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 2 4 4 8 8 8 16 16 32 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ count_omap_keys_with_filter cluster2 mirror rbd_mirror_leader image_map 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local filter=image_map 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ list_omap_keys cluster2 mirror rbd_mirror_leader 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local obj_name=rbd_mirror_leader 2026-02-22T22:08:30.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rados --cluster cluster2 -p mirror listomapkeys rbd_mirror_leader 2026-02-22T22:08:30.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -c image_map 2026-02-22T22:08:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 0 = 0 2026-02-22T22:08:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ error_code=0 2026-02-22T22:08:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set -e 2026-02-22T22:08:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:08:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T22:08:30.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: no blocklists' 2026-02-22T22:08:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:08:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:08:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:08:30 TEST: no blocklists 2026-02-22T22:08:30.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:08:30 TEST: no blocklists 2026-02-22T22:08:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS='--id admin' 2026-02-22T22:08:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster1 osd blocklist ls 2026-02-22T22:08:30.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -q 'listed 0 entries' 2026-02-22T22:08:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS='--id admin' 2026-02-22T22:08:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster2 osd blocklist ls 2026-02-22T22:08:30.598 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -q 'listed 0 entries' 2026-02-22T22:08:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:08:30.795 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ testlog 'TEST: force promote with a user snapshot' 2026-02-22T22:08:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ tee -a /tmp/tmp.S4fo03ccUK/rbd-mirror.test.log 2026-02-22T22:08:30.796 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ date '+%F %T' 2026-02-22T22:08:30.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo 2026-02-22 22:08:30 TEST: force promote with a user snapshot 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22 22:08:30 TEST: force promote with a user snapshot 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ force_promote_image=test_force_promote_user 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image_and_enable_mirror cluster2 mirror test_force_promote_user journal 10G 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local mode=journal 2026-02-22T22:08:30.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n journal ']' 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_image cluster2 mirror test_force_promote_user 10G 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=128 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 10G ']' 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=10G 2026-02-22T22:08:30.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:30.801 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 create --size 10G --image-feature layering,exclusive-lock,journaling mirror/test_force_promote_user 2026-02-22T22:08:30.831 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' pool = image ']' 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ case "${pool}" in 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 1 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:08:30.832 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:08:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:08:30.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:08:31.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:08:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:08:31.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:08:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-22T22:08:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:31.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T22:08:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:08:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:08:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:31.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:32.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T22:08:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:32.090 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:08:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:08:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:08:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:08:32.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T22:08:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T22:08:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:08:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:32.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:08:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:08:32.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:08:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:08:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:32.099 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:08:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:08:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:32.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:08:32.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:08:32.102 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:08:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:08:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:08:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:08:32.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:32.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:32.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:1 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:08:32.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:1 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:1 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:08:32.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:08:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:08:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:08:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:08:32.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:32.203 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:2 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:2 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:2 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T22:08:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:08:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:08:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:08:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:08:32.209 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:08:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:08:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:08:32.217 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:08:32.218 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:32.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:32.259 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:3 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:3 2026-02-22T22:08:32.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:3 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:08:32.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:08:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:08:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:08:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:08:32.274 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:32.315 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:32.316 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:08:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:08:33.317 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:33.321 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:33.322 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:08:33.323 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:08:33.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:08:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:08:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:08:33.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:33.378 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:08:33.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:08:33.379 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:08:33.380 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:08:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:08:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_started cluster1 mirror test_force_promote_user 2026-02-22T22:08:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:33.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=started 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:33.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user started 2026-02-22T22:08:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=started 2026-02-22T22:08:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:08:34.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:08:34.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:08:34.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:34.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:08:34.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:08:34.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:34.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:34.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:34.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:34.453 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:34.454 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:08:34.455 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:08:34.463 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:34.464 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:34.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:08:34.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:34.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:08:34.512 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:08:34.519 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:08:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:08:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:08:34.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:34.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:08:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:34.565 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:34.566 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:34.567 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:34.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:08:34.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:08:34.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:08:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:08:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:08:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:08:34.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:34.616 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:34.617 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:08:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:08:35.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:35.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:08:35.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:08:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:35.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:35.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:08:35.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:08:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:08:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:35.679 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:08:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:08:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:08:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:35.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:08:35.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:08:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:08:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:08:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:08:35.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: "state": "Replaying" 2026-02-22T22:08:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ current_state=started 2026-02-22T22:08:35.738 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test started = started 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror test_force_promote_user 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror test_force_promote_user 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local local_cluster=cluster1 2026-02-22T22:08:35.739 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:08:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:08:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:08:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:08:35.740 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:08:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_force_promote_user 2026-02-22T22:08:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:08:35.939 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_force_promote_user ']' 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_force_promote_user' 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:35.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:35.941 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:08:35.944 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:35.945 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:35.946 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:08:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:35.955 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:35.956 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:08:35.996 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:08:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:35.999 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:36.000 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:36.001 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:36.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:36.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:36.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:36.002 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:08:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:36.010 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_force_promote_user 2026-02-22T22:08:36.056 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_force_promote_user '' 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.057 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:08:36.058 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_force_promote_user 2026-02-22T22:08:36.059 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:08:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:08:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:08:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:08:36.084 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=99], [object_number=2, tag_tid=1, entry_tid=98]]], state=connected] 2026-02-22T22:08:36.085 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.086 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:08:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_force_promote_user 2026-02-22T22:08:36.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_force_promote_user '..*' 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:08:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_force_promote_user 2026-02-22T22:08:36.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:08:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:08:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:08:36.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:08:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=99], [object_number=2, tag_tid=1, entry_tid=98]]], state=connected] 2026-02-22T22:08:36.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' '!=' '' 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 'object_number=1, tag_tid=2, entry_tid=1' = '' 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_mirror_pos='object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:08:36.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:08:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_tag=2 2026-02-22T22:08:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:08:36.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:08:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:08:36.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_tag=2 2026-02-22T22:08:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:08:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:08:36.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:08:36.123 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_entry=3 2026-02-22T22:08:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=1' 2026-02-22T22:08:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:08:36.124 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cut -d= -f 2 2026-02-22T22:08:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_entry=1 2026-02-22T22:08:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test 2 = 2 -a 3 -le 1 2026-02-22T22:08:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ true 2026-02-22T22:08:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:08:36.125 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0.2 2026-02-22T22:08:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ flush cluster1 mirror test_force_promote_user 2026-02-22T22:08:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:08:36.327 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n test_force_promote_user ']' 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cmd='rbd mirror flush mirror/test_force_promote_user' 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemons cluster1 rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_instance=cluster1 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance=cluster1 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local loop_instance 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:08:36.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:08:36.331 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:36.332 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:36.333 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:08:36.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:36.342 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:36.383 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:08:36.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror flush : flush rbd mirror 2026-02-22T22:08:36.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:08:36.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ admin_daemon cluster1:1 rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:08:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ shift 2026-02-22T22:08:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:08:36.391 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:08:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:08:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:08:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:08:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:08:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:08:36.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:08:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:08:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:08:36.393 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:08:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:08:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:36.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:36.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:36.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:08:36.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:36.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:36.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:36.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror flush mirror/test_force_promote_user 2026-02-22T22:08:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:36.446 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_master_journal_position cluster2 mirror test_force_promote_user 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_force_promote_user '' 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local id_regexp= 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.447 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:08:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.448 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_force_promote_user 2026-02-22T22:08:36.449 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:08:36.474 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:08:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:08:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:08:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:08:36.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.476 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ master_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:08:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_mirror_journal_position cluster2 mirror test_force_promote_user 2026-02-22T22:08:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ get_journal_position cluster2 mirror test_force_promote_user '..*' 2026-02-22T22:08:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local pool=mirror 2026-02-22T22:08:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local image=test_force_promote_user 2026-02-22T22:08:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local 'id_regexp=..*' 2026-02-22T22:08:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ mkfname cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:08:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local status_log=/tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 journal status --image mirror/test_force_promote_user 2026-02-22T22:08:36.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ tee /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:minimum_set: 0 2026-02-22T22:08:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:active_set: 0 2026-02-22T22:08:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:registered clients: 2026-02-22T22:08:36.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:08:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: [id=5fac3253-137c-4f27-ad9d-6c8f7a882c9d, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0]]], state=connected] 2026-02-22T22:08:36.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.S4fo03ccUK/cluster2-mirror-test_force_promote_user.status 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 'object_number=3, tag_tid=2, entry_tid=3' -a 'object_number=3, tag_tid=2, entry_tid=3' = 'object_number=3, tag_tid=2, entry_tid=3' 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror test_force_promote_user 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = journal ']' 2026-02-22T22:08:36.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying primary_position 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:36.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+replaying primary_position '' 2026-02-22T22:08:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:37.510 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern=primary_position 2026-02-22T22:08:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:37.511 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-22T22:08:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote_user: 2026-02-22T22:08:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: c85a0e56-86a5-4719-8b99-6a174729422a 2026-02-22T22:08:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:08:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:08:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.1 on trial104 2026-02-22T22:08:37.543 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:35 2026-02-22T22:08:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:08:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:08:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:08:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:37' 2026-02-22T22:08:37.544 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:08:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+replaying 2026-02-22T22:08:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:08:37.545 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:08:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:08:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:08:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:08:37.546 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:08:37.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:08:37.547 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.1 on trial104 2026-02-22T22:08:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:08:37.548 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:08:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:35' 2026-02-22T22:08:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+replaying '2026-02-22 22:08:35' 2026-02-22T22:08:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:37.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+replaying 2026-02-22T22:08:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:08:35' 2026-02-22T22:08:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:37.550 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:08:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_force_promote_userc85a0e56-86a5-4719-8b99-6a174729422aup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}59545959mirror.1trial1042026-02-22 22:08:35cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:08:37' 2026-02-22T22:08:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:08:37.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T22:08:37.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:35' 2026-02-22T22:08:37.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T22:08:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+replaying 2026-02-22T22:08:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+replaying 2026-02-22T22:08:37.583 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+replaying 2026-02-22T22:08:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+replaying 2026-02-22T22:08:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:37.584 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:08:37.585 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:38.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-22T22:08:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote_user: 2026-02-22T22:08:38.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: c85a0e56-86a5-4719-8b99-6a174729422a 2026-02-22T22:08:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:08:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:08:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:08:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:37 2026-02-22T22:08:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:08:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T22:08:38.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+replaying 2026-02-22T22:08:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}} 2026-02-22T22:08:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:08:35' 2026-02-22T22:08:38.620 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:08:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T22:08:38.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:08:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T22:08:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:08:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:08:38.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:08:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:08:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:08:38.623 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:37' 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-22 22:08:37' 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:38.625 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:08:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:08:37' 2026-02-22T22:08:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:08:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:08:38.626 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:08:38.653 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK165996602mirror.0trial10419.2.3-689-gf90f4eaetrueOK66066609mirror.1trial10419.2.3-689-gf90f4eaefalseOK66136616mirror.2trial10419.2.3-689-gf90f4eaefalseOK66196625mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_force_promote_userc85a0e56-86a5-4719-8b99-6a174729422aup+stoppedlocal image is primary65996602mirror.0trial1042026-02-22 22:08:37cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{"entry_tid":101,"object_number":1,"tag_tid":1},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":2}}2026-02-22 22:08:35' 2026-02-22T22:08:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:08:38.654 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T22:08:38.655 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:08:37' 2026-02-22T22:08:38.656 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T22:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T22:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:08:38.657 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T22:08:38.658 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:08:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:38.659 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:08:38.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:08:38.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:08:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:08:38.705 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T22:08:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ create_snapshot cluster2 mirror test_force_promote_user snap1 2026-02-22T22:08:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap=snap1 2026-02-22T22:08:38.712 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 snap create mirror/test_force_promote_user@snap1 2026-02-22T22:08:39.349 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_force_promote_user 2560 4194304 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=2560 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size=4194304 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n 4194304 2026-02-22T22:08:39.355 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4194304 --io-threads 1 --io-total 10737418240 --io-pattern rand 2026-02-22T22:08:39.392 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4194304 io_threads 1 bytes 10737418240 pattern random 2026-02-22T22:08:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:08:40.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 1 201 202.409 810 MiB/s 2026-02-22T22:08:41.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 2 405 201.693 807 MiB/s 2026-02-22T22:08:42.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 3 604 200.667 803 MiB/s 2026-02-22T22:08:43.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 4 816 204.254 817 MiB/s 2026-02-22T22:08:44.404 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 5 1010 201.841 807 MiB/s 2026-02-22T22:08:45.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 6 1216 202.964 812 MiB/s 2026-02-22T22:08:46.398 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 7 1425 204.454 818 MiB/s 2026-02-22T22:08:47.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 8 1628 205.462 822 MiB/s 2026-02-22T22:08:48.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 9 1823 201.364 805 MiB/s 2026-02-22T22:08:49.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 10 2020 202.449 810 MiB/s 2026-02-22T22:08:50.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 11 2210 198.725 795 MiB/s 2026-02-22T22:08:51.394 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: 12 2404 195.961 784 MiB/s 2026-02-22T22:08:52.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 12 ops: 2560 ops/sec: 199.973 bytes/sec: 800 MiB/s 2026-02-22T22:08:52.204 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_snap_present cluster1 mirror test_force_promote_user snap1 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local snap_name=snap1 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 16 16 32 32 32 32 2026-02-22T22:08:52.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:53.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 info mirror/test_force_promote_user@snap1 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:rbd image 'test_force_promote_user': 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: size 10 GiB in 2560 objects 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: order 22 (4 MiB objects) 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: snapshot_count: 1 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: id: 174727367a45 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: block_name_prefix: rbd_data.174727367a45 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: format: 2 2026-02-22T22:08:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: op_features: 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: flags: 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: create_timestamp: Sun Feb 22 22:08:32 2026 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: access_timestamp: Sun Feb 22 22:08:32 2026 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: modify_timestamp: Sun Feb 22 22:08:32 2026 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: protected: False 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: journal: 174727367a45 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring state: enabled 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring mode: journal 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring global id: c85a0e56-86a5-4719-8b99-6a174729422a 2026-02-22T22:08:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: mirroring primary: false 2026-02-22T22:08:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:08:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:08:54.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster1 -KILL 2026-02-22T22:08:54.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:54.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig=-KILL 2026-02-22T22:08:54.237 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:08:54.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:54.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:0 -KILL 2026-02-22T22:08:54.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:08:54.239 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig=-KILL 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:0 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:54.240 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:08:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:08:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:08:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:08:54.241 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:08:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:08:54.250 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:08:54.251 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=90171 2026-02-22T22:08:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 90171 ']' 2026-02-22T22:08:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill -KILL 90171 2026-02-22T22:08:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:08:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:54.252 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.253 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90171 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:08:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90171 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:08:54.286 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:54.287 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:08:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:54.299 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.300 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:54.301 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:08:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:54.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:08:54.314 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:08:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:08:54.325 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:1 -KILL 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig=-KILL 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:08:54.328 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:1 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:08:54.329 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:08:54.330 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:08:54.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:08:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:08:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=90199 2026-02-22T22:08:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 90199 ']' 2026-02-22T22:08:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill -KILL 90199 2026-02-22T22:08:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:08:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90199 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:08:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90199 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:54.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:08:54.384 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:54.385 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:54.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:08:54.386 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.387 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:54.388 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:08:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:54.397 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:08:54.399 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:08:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:08:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:08:54.400 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:08:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:08:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:08:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:2 -KILL 2026-02-22T22:08:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:08:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig=-KILL 2026-02-22T22:08:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:54.410 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:08:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:08:54.411 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:2 2026-02-22T22:08:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:08:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:08:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:08:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:54.412 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:08:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:08:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:08:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:08:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:08:54.413 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:08:54.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:08:54.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:08:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=90243 2026-02-22T22:08:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 90243 ']' 2026-02-22T22:08:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill -KILL 90243 2026-02-22T22:08:54.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:08:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.424 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90243 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:08:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90243 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.457 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:08:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:08:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:08:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:08:54.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:08:54.467 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:08:54.469 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:08:54.470 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:08:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:54.471 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:08:54.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:08:54.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:08:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:08:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:08:54.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:08:54.482 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:08:54.483 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:08:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:08:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:08:54.493 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:3 -KILL 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig=-KILL 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:3 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:08:54.494 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:08:54.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:08:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:08:54.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:08:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=90302 2026-02-22T22:08:54.507 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 90302 ']' 2026-02-22T22:08:54.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill -KILL 90302 2026-02-22T22:08:54.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:08:54.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:08:54.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90302 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:08:54.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:08:54.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=90302 '$2 == pid {print; exit 1}' 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:08:54.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:08:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:54.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:08:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:08:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:08:54.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:54.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:08:54.561 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:08:54.562 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:08:54.563 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:08:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:08:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:08:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:08:54.564 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:08:54.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:08:54.573 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:08:54.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' journal = snapshot ']' 2026-02-22T22:08:54.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ promote_image cluster1 mirror test_force_promote_user --force 2026-02-22T22:08:54.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:08:54.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:08:54.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:08:54.574 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local force=--force 2026-02-22T22:08:54.575 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 mirror image promote mirror/test_force_promote_user --force 2026-02-22T22:09:04.612 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:Image promoted to primary 2026-02-22T22:09:04.618 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirrors cluster1 2026-02-22T22:09:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:04.619 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:0 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:0 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:09:04.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=0 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=0 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:04.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:09:04.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:04.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:1 2026-02-22T22:09:04.641 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:1 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=1 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:04.642 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=1 2026-02-22T22:09:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:04.643 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:09:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:2 2026-02-22T22:09:04.661 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:2 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=2 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=2 2026-02-22T22:09:04.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:04.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.2 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ start_mirror cluster1:3 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local val=cluster1:3 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster_var_name=cluster 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local instance_var_name=instance 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cluster=cluster1 2026-02-22T22:09:04.680 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ instance=3 2026-02-22T22:09:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval cluster=cluster1 2026-02-22T22:09:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ eval instance=3 2026-02-22T22:09:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:04.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd-mirror --cluster cluster1 --id mirror.3 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:09:04.699 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster1 mirror test_force_promote_user 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:09:04.700 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:09:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:04.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:09:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster1 mirror test_force_promote_user stopped 2026-02-22T22:09:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:05.701 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster1 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:05.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster1 2026-02-22T22:09:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:09:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:05.703 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:09:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:09:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:09:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:05.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:05.711 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:05.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:09:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:05.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:05.762 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:05.763 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:05.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:05.772 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.811 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:05.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:05.812 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:05.813 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:05.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:05.825 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:05.866 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:05.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:09:05.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:09:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:05.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 1 2026-02-22T22:09:06.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:09:06.925 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:09:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:09:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:06.928 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:09:06.929 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:06.930 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:06.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:06.940 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:06.981 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:06.982 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:06.985 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:06.986 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:06.987 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:06.994 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:06.995 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:07.034 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:07.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:07.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:07.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:07.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:07.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:07.047 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:07.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:07.087 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:07.088 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:09:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:09:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:09:07.100 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:07.101 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:07.140 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:07.141 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:07.142 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 2 2026-02-22T22:09:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:09:09.143 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:09:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:09:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:09.147 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:09.148 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:09.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:09.150 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:09.160 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:09.202 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:09.205 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:09.206 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:09.207 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:09.208 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:09.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:09.257 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:09:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:09.260 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:09.261 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:09.262 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:09.263 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:09.270 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:09.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:09.310 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:09.312 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:09:09.313 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:09:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:09.324 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:09.365 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:09.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 4 2026-02-22T22:09:13.367 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:09:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:09:13.371 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:13.372 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:13.373 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:13.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:13.381 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:13.382 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.422 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:13.423 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:13.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:13.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:13.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:13.429 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:13.430 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:13.436 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:13.475 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:09:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:13.478 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:09:13.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:13.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:13.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:13.487 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:13.488 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:13.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:13.526 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:09:13.529 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:13.530 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:13.531 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:09:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:09:13.532 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:09:13.538 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:13.539 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:13.578 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:13.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 8 2026-02-22T22:09:21.579 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:09:21.580 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:09:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:09:21.581 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:21.582 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:21.593 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:21.594 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:21.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:21.634 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:09:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:21.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:21.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:21.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:21.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:21.689 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:21.690 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:21.691 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:21.702 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:21.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:21.743 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:3 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:21.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:3 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:21.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:09:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:09:21.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:09:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:21.756 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:21.797 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:21.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:21.798 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 8 2026-02-22T22:09:29.799 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:09:29.800 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:29.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:29.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:29.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:29.814 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:29.855 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:1 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:29.859 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:29.860 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:29.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:29.868 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:29.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:29.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:29.869 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:Invalid command: unused arguments: ['mirror/test_force_promote_user'] 2026-02-22T22:09:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:09:29.910 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:admin_socket: invalid command 2026-02-22T22:09:29.913 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster1:2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:29.914 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster1:2 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:09:29.915 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster1 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:29.916 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:29.923 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:29.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:29.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:29.924 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:29.971 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:09:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:09:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:09:29.972 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:09:29.973 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_stopped cluster2 mirror test_force_promote_user 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state=stopped 2026-02-22T22:09:29.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local s 2026-02-22T22:09:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:29.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:09:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_image_replay_state cluster2 mirror test_force_promote_user stopped 2026-02-22T22:09:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local test_state=stopped 2026-02-22T22:09:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status_result 2026-02-22T22:09:30.974 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local current_state=stopped 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemons cluster2 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_instance=cluster2 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance=cluster2 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local loop_instance 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sleep 0 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ grep -i state 2026-02-22T22:09:30.975 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:09:30.976 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ seq 0 3 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ shift 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:09:30.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:09:30.978 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ local_cluster=cluster2 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:30.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:09:30.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:09:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:09:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:09:30.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/test_force_promote_user 2026-02-22T22:09:31.037 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ return 0 2026-02-22T22:09:31.038 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status_result=' "state": "Stopped"' 2026-02-22T22:09:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo ' "state": "Stopped"' 2026-02-22T22:09:31.039 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep -i Replaying 2026-02-22T22:09:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test stopped = stopped 2026-02-22T22:09:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped 2026-02-22T22:09:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:31.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:09:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:09:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:09:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:31.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster1 mirror test_force_promote_user up+stopped '' '' 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:09:32.040 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:09:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:09:32.041 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror image status mirror/test_force_promote_user 2026-02-22T22:09:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote_user: 2026-02-22T22:09:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: c85a0e56-86a5-4719-8b99-6a174729422a 2026-02-22T22:09:32.074 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.2 on trial104 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:09:29 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster2 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:09:27' 2026-02-22T22:09:32.075 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:09:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T22:09:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:09:32.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T22:09:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:09:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:09:32.077 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:09:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:09:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:09:32.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.2 on trial104 2026-02-22T22:09:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:09:32.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:09:29' 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster1 mirror test_force_promote_user up+stopped '2026-02-22 22:09:29' 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:09:29' 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:09:32.081 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format xml 2026-02-22T22:09:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK159505985mirror.0trial10419.2.3-689-gf90f4eaefalseOK59545959mirror.1trial10419.2.3-689-gf90f4eaetrueOK59615966mirror.2trial10419.2.3-689-gf90f4eaefalseOK59685971mirror.3trial10419.2.3-689-gf90f4eaefalseOK62286233mirror.0trial10419.2.3-689-gf90f4eaefalseOK62356239mirror.1trial10419.2.3-689-gf90f4eaefalseOK62416246mirror.2trial10419.2.3-689-gf90f4eaetrueOK62486252mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_force_promote_userc85a0e56-86a5-4719-8b99-6a174729422aup+stoppedlocal image is primary62416246mirror.2trial1042026-02-22 22:09:29cluster2605c498e-0476-4c03-b186-dac6ca3f245bup+stoppedlocal image is primary2026-02-22 22:09:27' 2026-02-22T22:09:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:09:32.113 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T22:09:32.114 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:09:29' 2026-02-22T22:09:32.115 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T22:09:32.116 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T22:09:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:09:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T22:09:32.117 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:09:32.118 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ wait_for_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:09:32.119 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:09:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_status_in_pool_dir cluster2 mirror test_force_promote_user up+stopped '' '' 2026-02-22T22:09:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:09:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local description_pattern= 2026-02-22T22:09:33.120 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local service_pattern= 2026-02-22T22:09:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:09:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:09:33.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror image status mirror/test_force_promote_user 2026-02-22T22:09:33.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='test_force_promote_user: 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: global_id: c85a0e56-86a5-4719-8b99-6a174729422a 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: service: mirror.0 on trial104 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:09:32 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: peer_sites: 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: name: cluster1 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: state: up+stopped 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: description: local image is primary 2026-02-22T22:09:33.153 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: last_update: 2026-02-22 22:09:29' 2026-02-22T22:09:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ state: .*up+stopped' 2026-02-22T22:09:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: state: up+stopped 2026-02-22T22:09:33.154 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^ description: .*' 2026-02-22T22:09:33.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: description: local image is primary 2026-02-22T22:09:33.155 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n '' ']' 2026-02-22T22:09:33.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:09:33.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep '^up+' 2026-02-22T22:09:33.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:09:33.156 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:09:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: service: mirror.0 on trial104 2026-02-22T22:09:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update 2026-02-22T22:09:33.157 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:09:32' 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test_mirror_pool_status_verbose cluster2 mirror test_force_promote_user up+stopped '2026-02-22 22:09:32' 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local state_pattern=up+stopped 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local 'prev_last_update=2026-02-22 22:09:32' 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local status 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ CEPH_ARGS= 2026-02-22T22:09:33.159 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:09:33.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ status='OKOKOK165996602mirror.0trial10419.2.3-689-gf90f4eaetrueOK66066609mirror.1trial10419.2.3-689-gf90f4eaefalseOK66136616mirror.2trial10419.2.3-689-gf90f4eaefalseOK66196625mirror.3trial10419.2.3-689-gf90f4eaefalseOKtest_force_promote_userc85a0e56-86a5-4719-8b99-6a174729422aup+stoppedlocal image is primary65996602mirror.0trial1042026-02-22 22:09:32cluster15fac3253-137c-4f27-ad9d-6c8f7a882c9dup+stoppedlocal image is primary2026-02-22 22:09:29' 2026-02-22T22:09:33.188 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local last_update state 2026-02-22T22:09:33.189 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/last_update' 2026-02-22T22:09:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ last_update='2026-02-22 22:09:32' 2026-02-22T22:09:33.190 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''test_force_promote_user'\'']/state' 2026-02-22T22:09:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ state=up+stopped 2026-02-22T22:09:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo up+stopped 2026-02-22T22:09:33.192 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ grep up+stopped 2026-02-22T22:09:33.193 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:up+stopped 2026-02-22T22:09:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster1 mirror test_force_promote_user 100 2026-02-22T22:09:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:33.194 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:09:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:09:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:09:33.195 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster1 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:09:33.338 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:09:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:09:33.350 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 8333.5 bytes/sec: 33 MiB/s 2026-02-22T22:09:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ write_image cluster2 mirror test_force_promote_user 100 2026-02-22T22:09:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local count=100 2026-02-22T22:09:33.357 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local size= 2026-02-22T22:09:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ size=4096 2026-02-22T22:09:33.358 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster cluster2 bench mirror/test_force_promote_user --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:09:33.421 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:09:33.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:09:33.434 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:elapsed: 0 ops: 100 ops/sec: 7692.46 bytes/sec: 30 MiB/s 2026-02-22T22:09:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster1 mirror test_force_promote_user 2026-02-22T22:09:33.441 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:33.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster1 mirror test_force_promote_user 2026-02-22T22:09:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:33.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 snap purge mirror/test_force_promote_user 2026-02-22T22:09:34.340 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T22:09:34.345 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster1 rm mirror/test_force_promote_user 2026-02-22T22:09:34.678 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:09:34.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:34.681 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image_retry cluster2 mirror test_force_promote_user 2026-02-22T22:09:34.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:34.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:34.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:34.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:34.682 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_force_promote_user 2026-02-22T22:09:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:34.683 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote_user 2026-02-22T22:09:35.366 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T22:09:35.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote_user 2026-02-22T22:09:35.764 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 0% complete...failed. 2026-02-22T22:09:35.766 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:rbd: image has snapshots - these must be deleted with 'rbd snap purge' before the image can be removed. 2026-02-22T22:09:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:35.768 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:09:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ remove_image cluster2 mirror test_force_promote_user 2026-02-22T22:09:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pool=mirror 2026-02-22T22:09:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local image=test_force_promote_user 2026-02-22T22:09:36.770 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 snap purge mirror/test_force_promote_user 2026-02-22T22:09:37.369 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2026-02-22T22:09:37.374 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rbd --cluster=cluster2 rm mirror/test_force_promote_user 2026-02-22T22:09:38.045 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2026-02-22T22:09:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ return 0 2026-02-22T22:09:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ cleanup 0 2026-02-22T22:09:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local error_code=0 2026-02-22T22:09:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ set +e 2026-02-22T22:09:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -ne 0 ']' 2026-02-22T22:09:38.049 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z '' ']' 2026-02-22T22:09:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster instance 2026-02-22T22:09:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:09:38.050 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:09:38.390 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror' does not exist 2026-02-22T22:09:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:09:38.396 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:09:39.407 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror' does not exist 2026-02-22T22:09:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:09:39.415 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:09:40.395 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:09:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ CEPH_ARGS= 2026-02-22T22:09:40.402 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:09:40.651 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:2026-02-22T22:09:40.650+0000 7f1bccd0b640 0 --2- 10.20.193.104:0/3434831150 >> [v2:10.20.193.104:3300/0,v1:10.20.193.104:6789/0] conn(0x7f1bc80a25a0 0x7f1bc8149780 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request get_initial_auth_request returned -2 2026-02-22T22:09:41.417 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:09:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:09:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster1 2026-02-22T22:09:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1 2026-02-22T22:09:41.425 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:41.426 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:0 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:41.427 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:0 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:0 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:41.428 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:09:41.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:09:41.440 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:09:41.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=95658 2026-02-22T22:09:41.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 95658 ']' 2026-02-22T22:09:41.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 95658 2026-02-22T22:09:41.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:41.442 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:41.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:41.443 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95658 '$2 == pid {print; exit 1}' 2026-02-22T22:09:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:41.461 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95658 '$2 == pid {print; exit 1}' 2026-02-22T22:09:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:09:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:09:41.479 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:41.480 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:41.481 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:41.492 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:09:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:09:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:41.495 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:41.496 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:41.497 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:09:41.505 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:41.506 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:0 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:0 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:09:41.508 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:41.509 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:09:41.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:09:41.520 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:1 '' 2026-02-22T22:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:1 2026-02-22T22:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:41.522 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:1 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:1 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:1 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:09:41.523 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:41.524 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:09:41.533 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:09:41.534 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:09:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=95702 2026-02-22T22:09:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 95702 ']' 2026-02-22T22:09:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 95702 2026-02-22T22:09:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:41.535 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:41.536 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95702 '$2 == pid {print; exit 1}' 2026-02-22T22:09:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 95702 0.2 0.0 474216 40104 ? Ssl 22:09 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:09:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:41.551 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 1 2026-02-22T22:09:42.552 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:42.553 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95702 '$2 == pid {print; exit 1}' 2026-02-22T22:09:42.568 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 95702 0.2 0.0 474216 40108 ? Ssl 22:09 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:09:42.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:42.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 2 2026-02-22T22:09:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:44.569 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95702 '$2 == pid {print; exit 1}' 2026-02-22T22:09:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:ubuntu 95702 0.2 0.0 474216 40120 ? Ssl 22:09 0:00 rbd-mirror --cluster cluster1 --id mirror.1 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:09:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:44.586 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 4 2026-02-22T22:09:48.587 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.588 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95702 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.605 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95702 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster1 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:48.621 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:48.622 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:48.633 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:1 cluster2 2026-02-22T22:09:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:1 2026-02-22T22:09:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:48.635 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 local_cluster instance 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:48.636 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:48.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:48.637 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'admin socket' 2026-02-22T22:09:48.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:48.647 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.asok 2026-02-22T22:09:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:1 2026-02-22T22:09:48.648 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:1 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:1 cluster instance 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:1 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster1:1 ']' 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:48.649 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:48.650 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.1 'pid file' 2026-02-22T22:09:48.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:09:48.660 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.1.pid 2026-02-22T22:09:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:2 '' 2026-02-22T22:09:48.662 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:2 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:2 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:2 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:2 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:48.663 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:09:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:09:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:48.664 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:09:48.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:09:48.674 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:09:48.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=95762 2026-02-22T22:09:48.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 95762 ']' 2026-02-22T22:09:48.675 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 95762 2026-02-22T22:09:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.676 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95762 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.692 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.693 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95762 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster1 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.709 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:48.710 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:48.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:48.719 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:2 cluster2 2026-02-22T22:09:48.721 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:2 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 local_cluster instance 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:48.722 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:48.723 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'admin socket' 2026-02-22T22:09:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:48.733 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.asok 2026-02-22T22:09:48.734 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:2 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:2 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:2 cluster instance 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:2 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster1:2 ']' 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:48.735 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:48.736 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.2 'pid file' 2026-02-22T22:09:48.745 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:09:48.746 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.2.pid 2026-02-22T22:09:48.747 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster1:3 '' 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster1:3 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster1:3 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster1:3 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster1:3 2026-02-22T22:09:48.748 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster1 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster1 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:09:48.749 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:09:48.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:09:48.758 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:09:48.759 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=95822 2026-02-22T22:09:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 95822 ']' 2026-02-22T22:09:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 95822 2026-02-22T22:09:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:48.760 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.761 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95822 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.776 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=95822 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster1 2026-02-22T22:09:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:09:48.791 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:48.792 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:48.793 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:09:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:48.803 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:48.804 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster1:3 cluster2 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster1:3 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 local_cluster instance 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.805 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster1 2026-02-22T22:09:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster1 2026-02-22T22:09:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:48.806 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'admin socket' 2026-02-22T22:09:48.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:48.815 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.asok 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster1:3 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1:3 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster1:3 cluster instance 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster1:3 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.817 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster1 2026-02-22T22:09:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster1:3 ']' 2026-02-22T22:09:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster1 2026-02-22T22:09:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster1 2026-02-22T22:09:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:48.818 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.3 'pid file' 2026-02-22T22:09:48.828 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:09:48.829 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.3.pid 2026-02-22T22:09:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:09:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirrors cluster2 2026-02-22T22:09:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2 2026-02-22T22:09:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:48.830 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ seq 0 3 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:0 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:0 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:48.833 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:0 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=0 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=0 2026-02-22T22:09:48.834 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:09:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:09:48.843 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:09:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=94109 2026-02-22T22:09:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 94109 ']' 2026-02-22T22:09:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 94109 2026-02-22T22:09:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.845 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94109 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:48.861 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.862 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94109 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T22:09:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:09:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:48.878 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:48.879 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:48.880 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:09:48.889 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:09:48.890 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:09:48.891 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:09:48.892 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:48.893 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:09:48.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:09:48.903 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:09:48.905 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:0 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:0 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:0 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=0 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:09:48.906 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:09:48.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:48.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=0 2026-02-22T22:09:48.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=0 2026-02-22T22:09:48.907 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:09:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:09:48.917 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:1 '' 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:1 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:1 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:1 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:48.920 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:1 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 1 = cluster2:1 ']' 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=1 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=1 2026-02-22T22:09:48.921 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T22:09:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:09:48.931 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:09:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=94153 2026-02-22T22:09:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 94153 ']' 2026-02-22T22:09:48.932 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 94153 2026-02-22T22:09:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.933 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94153 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.948 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:48.949 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94153 '$2 == pid {print; exit 1}' 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster1 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:48.965 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:09:48.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:09:48.977 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:09:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:1 cluster2 2026-02-22T22:09:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:1 2026-02-22T22:09:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.979 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 local_cluster instance 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:48.980 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'admin socket' 2026-02-22T22:09:48.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:09:48.990 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.asok 2026-02-22T22:09:48.991 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:1 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:1 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:1 cluster instance 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:1 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=1 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 1 = cluster2:1 ']' 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:09:48.992 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=1 2026-02-22T22:09:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=1 2026-02-22T22:09:48.993 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.1 'pid file' 2026-02-22T22:09:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:09:49.003 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.1.pid 2026-02-22T22:09:49.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:49.005 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:2 '' 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:2 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:2 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:2 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:2 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:49.006 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 2 = cluster2:2 ']' 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=2 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=2 2026-02-22T22:09:49.007 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T22:09:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:09:49.017 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:09:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=94213 2026-02-22T22:09:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 94213 ']' 2026-02-22T22:09:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 94213 2026-02-22T22:09:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:49.019 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:49.020 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94213 '$2 == pid {print; exit 1}' 2026-02-22T22:09:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:49.035 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94213 '$2 == pid {print; exit 1}' 2026-02-22T22:09:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster1 2026-02-22T22:09:49.051 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:49.052 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:49.053 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:09:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:09:49.063 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:09:49.065 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:2 cluster2 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:2 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 local_cluster instance 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:49.066 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:09:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:49.067 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'admin socket' 2026-02-22T22:09:49.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:09:49.076 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.asok 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:2 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:2 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:2 cluster instance 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:2 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=2 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 2 = cluster2:2 ']' 2026-02-22T22:09:49.078 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:09:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=2 2026-02-22T22:09:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=2 2026-02-22T22:09:49.079 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.2 'pid file' 2026-02-22T22:09:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:09:49.089 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.2.pid 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ stop_mirror cluster2:3 '' 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local cluster=cluster2:3 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local sig= 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test -n '' 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ local pid 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ daemon_pid_file cluster2:3 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster=cluster2:3 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance 2026-02-22T22:09:49.092 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local val=cluster2:3 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local instance_var_name=instance 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ '[' 3 = cluster2:3 ']' 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval cluster=cluster2 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ cluster=cluster2 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ eval instance=3 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ instance=3 2026-02-22T22:09:49.093 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T22:09:49.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:09:49.103 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:09:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ pid=94275 2026-02-22T22:09:49.104 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -n 94275 ']' 2026-02-22T22:09:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ kill 94275 2026-02-22T22:09:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:09:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ sleep 0 2026-02-22T22:09:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:49.105 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94275 '$2 == pid {print; exit 1}' 2026-02-22T22:09:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ break 2026-02-22T22:09:49.121 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ ps auxww 2026-02-22T22:09:49.122 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ awk -v pid=94275 '$2 == pid {print; exit 1}' 2026-02-22T22:09:49.137 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster1 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster1 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:49.138 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:49.139 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:09:49.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:09:49.149 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_asok_file cluster2:3 cluster2 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local local_cluster=cluster2:3 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 local_cluster instance 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:49.151 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:09:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval local_cluster=cluster2 2026-02-22T22:09:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ local_cluster=cluster2 2026-02-22T22:09:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:49.152 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'admin socket' 2026-02-22T22:09:49.161 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:09:49.162 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.asok 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ daemon_pid_file cluster2:3 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster=cluster2:3 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ set_cluster_instance cluster2:3 cluster instance 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local val=cluster2:3 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local cluster_var_name=cluster 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ local instance_var_name=instance 2026-02-22T22:09:49.164 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ cluster=cluster2 2026-02-22T22:09:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ instance=3 2026-02-22T22:09:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ '[' 3 = cluster2:3 ']' 2026-02-22T22:09:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval cluster=cluster2 2026-02-22T22:09:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ cluster=cluster2 2026-02-22T22:09:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ eval instance=3 2026-02-22T22:09:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ instance=3 2026-02-22T22:09:49.165 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.3 'pid file' 2026-02-22T22:09:49.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:09:49.175 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.3.pid 2026-02-22T22:09:49.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' -z 1 ']' 2026-02-22T22:09:49.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ test '' = /tmp/tmp.S4fo03ccUK 2026-02-22T22:09:49.176 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ rm -Rf /tmp/tmp.S4fo03ccUK 2026-02-22T22:09:49.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:09:49.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ echo OK 2026-02-22T22:09:49.177 INFO:tasks.workunit.cluster1.client.mirror.trial104.stdout:OK 2026-02-22T22:09:49.178 INFO:tasks.workunit.cluster1.client.mirror.trial104.stderr:+ exit 0 2026-02-22T22:09:49.179 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-22T22:09:49.179 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-22T22:09:49.247 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror.sh'] on cluster1.client.mirror... 2026-02-22T22:09:49.247 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-22T22:09:49.587 DEBUG:teuthology.parallel:result is None 2026-02-22T22:09:49.587 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:09:49.613 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:09:49.613 DEBUG:teuthology.orchestra.run.trial104:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:09:49.670 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:09:49.670 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:09:49.735 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:09:49.735 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:09:49.888 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T22:09:49.889 INFO:teuthology.orchestra.run.trial104.stderr:dumped all 2026-02-22T22:09:49.897 INFO:teuthology.orchestra.run.trial104.stdout:{"pg_ready":true,"pg_map":{"version":880,"stamp":"2026-02-22T22:09:49.301144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":108,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":796436,"kb_used_data":5080,"kb_used_omap":4,"kb_used_meta":791291,"kb_avail":282319084,"statfs":{"total":289910292480,"available":289094742016,"internally_reserved":0,"allocated":5201920,"data_stored":3172675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":810282323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-18,"num_object_clones":0,"num_object_copies":-36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-18,"num_whiteouts":0,"num_read":-7975,"num_read_kb":-14122,"num_write":-971,"num_write_kb":-2372,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.941017"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.368184+0000","last_change":"2026-02-22T21:45:47.055786+0000","last_active":"2026-02-22T22:09:41.368184+0000","last_peered":"2026-02-22T22:09:41.368184+0000","last_clean":"2026-02-22T22:09:41.368184+0000","last_became_active":"2026-02-22T21:45:45.053797+0000","last_became_peered":"2026-02-22T21:45:45.053797+0000","last_unstale":"2026-02-22T22:09:41.368184+0000","last_undegraded":"2026-02-22T22:09:41.368184+0000","last_fullsized":"2026-02-22T22:09:41.368184+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:53:17.577669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050143900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370673+0000","last_change":"2026-02-22T21:45:47.055663+0000","last_active":"2026-02-22T22:09:41.370673+0000","last_peered":"2026-02-22T22:09:41.370673+0000","last_clean":"2026-02-22T22:09:41.370673+0000","last_became_active":"2026-02-22T21:45:45.053799+0000","last_became_peered":"2026-02-22T21:45:45.053799+0000","last_unstale":"2026-02-22T22:09:41.370673+0000","last_undegraded":"2026-02-22T22:09:41.370673+0000","last_fullsized":"2026-02-22T22:09:41.370673+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:27:16.854273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0003992,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370478+0000","last_change":"2026-02-22T21:45:47.055898+0000","last_active":"2026-02-22T22:09:41.370478+0000","last_peered":"2026-02-22T22:09:41.370478+0000","last_clean":"2026-02-22T22:09:41.370478+0000","last_became_active":"2026-02-22T21:45:45.053506+0000","last_became_peered":"2026-02-22T21:45:45.053506+0000","last_unstale":"2026-02-22T22:09:41.370478+0000","last_undegraded":"2026-02-22T22:09:41.370478+0000","last_fullsized":"2026-02-22T22:09:41.370478+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:10:14.553071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060315800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.367026+0000","last_change":"2026-02-22T21:45:47.055731+0000","last_active":"2026-02-22T22:09:41.367026+0000","last_peered":"2026-02-22T22:09:41.367026+0000","last_clean":"2026-02-22T22:09:41.367026+0000","last_became_active":"2026-02-22T21:45:45.053788+0000","last_became_peered":"2026-02-22T21:45:45.053788+0000","last_unstale":"2026-02-22T22:09:41.367026+0000","last_undegraded":"2026-02-22T22:09:41.367026+0000","last_fullsized":"2026-02-22T22:09:41.367026+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:15:31.306502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048115199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":312,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.456266+0000","last_change":"2026-02-22T21:45:47.057685+0000","last_active":"2026-02-22T22:09:41.456266+0000","last_peered":"2026-02-22T22:09:41.456266+0000","last_clean":"2026-02-22T22:09:41.456266+0000","last_became_active":"2026-02-22T21:45:45.053848+0000","last_became_peered":"2026-02-22T21:45:45.053848+0000","last_unstale":"2026-02-22T22:09:41.456266+0000","last_undegraded":"2026-02-22T22:09:41.456266+0000","last_fullsized":"2026-02-22T22:09:41.456266+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:30:42.167683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089195400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":308,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.457119+0000","last_change":"2026-02-22T21:45:47.106335+0000","last_active":"2026-02-22T22:09:41.457119+0000","last_peered":"2026-02-22T22:09:41.457119+0000","last_clean":"2026-02-22T22:09:41.457119+0000","last_became_active":"2026-02-22T21:45:45.053963+0000","last_became_peered":"2026-02-22T21:45:45.053963+0000","last_unstale":"2026-02-22T22:09:41.457119+0000","last_undegraded":"2026-02-22T22:09:41.457119+0000","last_fullsized":"2026-02-22T22:09:41.457119+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:40:42.243266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000224897,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.455250+0000","last_change":"2026-02-22T21:45:47.106341+0000","last_active":"2026-02-22T22:09:41.455250+0000","last_peered":"2026-02-22T22:09:41.455250+0000","last_clean":"2026-02-22T22:09:41.455250+0000","last_became_active":"2026-02-22T21:45:45.053834+0000","last_became_peered":"2026-02-22T21:45:45.053834+0000","last_unstale":"2026-02-22T22:09:41.455250+0000","last_undegraded":"2026-02-22T22:09:41.455250+0000","last_fullsized":"2026-02-22T22:09:41.455250+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:44:15.059855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000242379,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":310,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.369837+0000","last_change":"2026-02-22T21:45:47.056034+0000","last_active":"2026-02-22T22:09:41.369837+0000","last_peered":"2026-02-22T22:09:41.369837+0000","last_clean":"2026-02-22T22:09:41.369837+0000","last_became_active":"2026-02-22T21:45:45.053434+0000","last_became_peered":"2026-02-22T21:45:45.053434+0000","last_unstale":"2026-02-22T22:09:41.369837+0000","last_undegraded":"2026-02-22T22:09:41.369837+0000","last_fullsized":"2026-02-22T22:09:41.369837+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:45.436732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076235300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":400,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.364574+0000","last_change":"2026-02-22T21:45:41.044949+0000","last_active":"2026-02-22T22:09:41.364574+0000","last_peered":"2026-02-22T22:09:41.364574+0000","last_clean":"2026-02-22T22:09:41.364574+0000","last_became_active":"2026-02-22T21:45:41.044711+0000","last_became_peered":"2026-02-22T21:45:41.044711+0000","last_unstale":"2026-02-22T22:09:41.364574+0000","last_undegraded":"2026-02-22T22:09:41.364574+0000","last_fullsized":"2026-02-22T22:09:41.364574+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_clean_scrub_stamp":"2026-02-22T21:45:40.040843+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:25:57.694568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705956,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":239212,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":237886,"kb_avail":94132628,"statfs":{"total":96636764160,"available":96391811072,"internally_reserved":0,"allocated":1335296,"data_stored":663967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243595719},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705958,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":285556,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":283646,"kb_avail":94086284,"statfs":{"total":96636764160,"available":96344354816,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290453958},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738663,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":271668,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":269758,"kb_avail":94100172,"statfs":{"total":96636764160,"available":96358576128,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":276232646},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:09:49.898 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:09:50.055 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T22:09:50.056 INFO:teuthology.orchestra.run.trial104.stderr:dumped all 2026-02-22T22:09:50.064 INFO:teuthology.orchestra.run.trial104.stdout:{"pg_ready":true,"pg_map":{"version":880,"stamp":"2026-02-22T22:09:49.301144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":108,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":796436,"kb_used_data":5080,"kb_used_omap":4,"kb_used_meta":791291,"kb_avail":282319084,"statfs":{"total":289910292480,"available":289094742016,"internally_reserved":0,"allocated":5201920,"data_stored":3172675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":810282323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-18,"num_object_clones":0,"num_object_copies":-36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-18,"num_whiteouts":0,"num_read":-7975,"num_read_kb":-14122,"num_write":-971,"num_write_kb":-2372,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.941017"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.368184+0000","last_change":"2026-02-22T21:45:47.055786+0000","last_active":"2026-02-22T22:09:41.368184+0000","last_peered":"2026-02-22T22:09:41.368184+0000","last_clean":"2026-02-22T22:09:41.368184+0000","last_became_active":"2026-02-22T21:45:45.053797+0000","last_became_peered":"2026-02-22T21:45:45.053797+0000","last_unstale":"2026-02-22T22:09:41.368184+0000","last_undegraded":"2026-02-22T22:09:41.368184+0000","last_fullsized":"2026-02-22T22:09:41.368184+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:53:17.577669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050143900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370673+0000","last_change":"2026-02-22T21:45:47.055663+0000","last_active":"2026-02-22T22:09:41.370673+0000","last_peered":"2026-02-22T22:09:41.370673+0000","last_clean":"2026-02-22T22:09:41.370673+0000","last_became_active":"2026-02-22T21:45:45.053799+0000","last_became_peered":"2026-02-22T21:45:45.053799+0000","last_unstale":"2026-02-22T22:09:41.370673+0000","last_undegraded":"2026-02-22T22:09:41.370673+0000","last_fullsized":"2026-02-22T22:09:41.370673+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:27:16.854273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0003992,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370478+0000","last_change":"2026-02-22T21:45:47.055898+0000","last_active":"2026-02-22T22:09:41.370478+0000","last_peered":"2026-02-22T22:09:41.370478+0000","last_clean":"2026-02-22T22:09:41.370478+0000","last_became_active":"2026-02-22T21:45:45.053506+0000","last_became_peered":"2026-02-22T21:45:45.053506+0000","last_unstale":"2026-02-22T22:09:41.370478+0000","last_undegraded":"2026-02-22T22:09:41.370478+0000","last_fullsized":"2026-02-22T22:09:41.370478+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:10:14.553071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060315800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.367026+0000","last_change":"2026-02-22T21:45:47.055731+0000","last_active":"2026-02-22T22:09:41.367026+0000","last_peered":"2026-02-22T22:09:41.367026+0000","last_clean":"2026-02-22T22:09:41.367026+0000","last_became_active":"2026-02-22T21:45:45.053788+0000","last_became_peered":"2026-02-22T21:45:45.053788+0000","last_unstale":"2026-02-22T22:09:41.367026+0000","last_undegraded":"2026-02-22T22:09:41.367026+0000","last_fullsized":"2026-02-22T22:09:41.367026+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:15:31.306502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048115199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":312,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.456266+0000","last_change":"2026-02-22T21:45:47.057685+0000","last_active":"2026-02-22T22:09:41.456266+0000","last_peered":"2026-02-22T22:09:41.456266+0000","last_clean":"2026-02-22T22:09:41.456266+0000","last_became_active":"2026-02-22T21:45:45.053848+0000","last_became_peered":"2026-02-22T21:45:45.053848+0000","last_unstale":"2026-02-22T22:09:41.456266+0000","last_undegraded":"2026-02-22T22:09:41.456266+0000","last_fullsized":"2026-02-22T22:09:41.456266+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:30:42.167683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089195400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":308,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.457119+0000","last_change":"2026-02-22T21:45:47.106335+0000","last_active":"2026-02-22T22:09:41.457119+0000","last_peered":"2026-02-22T22:09:41.457119+0000","last_clean":"2026-02-22T22:09:41.457119+0000","last_became_active":"2026-02-22T21:45:45.053963+0000","last_became_peered":"2026-02-22T21:45:45.053963+0000","last_unstale":"2026-02-22T22:09:41.457119+0000","last_undegraded":"2026-02-22T22:09:41.457119+0000","last_fullsized":"2026-02-22T22:09:41.457119+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:40:42.243266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000224897,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.455250+0000","last_change":"2026-02-22T21:45:47.106341+0000","last_active":"2026-02-22T22:09:41.455250+0000","last_peered":"2026-02-22T22:09:41.455250+0000","last_clean":"2026-02-22T22:09:41.455250+0000","last_became_active":"2026-02-22T21:45:45.053834+0000","last_became_peered":"2026-02-22T21:45:45.053834+0000","last_unstale":"2026-02-22T22:09:41.455250+0000","last_undegraded":"2026-02-22T22:09:41.455250+0000","last_fullsized":"2026-02-22T22:09:41.455250+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:44:15.059855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000242379,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":310,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.369837+0000","last_change":"2026-02-22T21:45:47.056034+0000","last_active":"2026-02-22T22:09:41.369837+0000","last_peered":"2026-02-22T22:09:41.369837+0000","last_clean":"2026-02-22T22:09:41.369837+0000","last_became_active":"2026-02-22T21:45:45.053434+0000","last_became_peered":"2026-02-22T21:45:45.053434+0000","last_unstale":"2026-02-22T22:09:41.369837+0000","last_undegraded":"2026-02-22T22:09:41.369837+0000","last_fullsized":"2026-02-22T22:09:41.369837+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:45.436732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076235300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":400,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.364574+0000","last_change":"2026-02-22T21:45:41.044949+0000","last_active":"2026-02-22T22:09:41.364574+0000","last_peered":"2026-02-22T22:09:41.364574+0000","last_clean":"2026-02-22T22:09:41.364574+0000","last_became_active":"2026-02-22T21:45:41.044711+0000","last_became_peered":"2026-02-22T21:45:41.044711+0000","last_unstale":"2026-02-22T22:09:41.364574+0000","last_undegraded":"2026-02-22T22:09:41.364574+0000","last_fullsized":"2026-02-22T22:09:41.364574+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_clean_scrub_stamp":"2026-02-22T21:45:40.040843+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:25:57.694568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705956,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":239212,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":237886,"kb_avail":94132628,"statfs":{"total":96636764160,"available":96391811072,"internally_reserved":0,"allocated":1335296,"data_stored":663967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243595719},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705958,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":285556,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":283646,"kb_avail":94086284,"statfs":{"total":96636764160,"available":96344354816,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290453958},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738663,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":271668,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":269758,"kb_avail":94100172,"statfs":{"total":96636764160,"available":96358576128,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":276232646},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:09:50.064 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:09:50.064 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:09:50.221 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T22:09:50.222 INFO:teuthology.orchestra.run.trial104.stderr:dumped all 2026-02-22T22:09:50.230 INFO:teuthology.orchestra.run.trial104.stdout:{"pg_ready":true,"pg_map":{"version":880,"stamp":"2026-02-22T22:09:49.301144+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":108,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":796436,"kb_used_data":5080,"kb_used_omap":4,"kb_used_meta":791291,"kb_avail":282319084,"statfs":{"total":289910292480,"available":289094742016,"internally_reserved":0,"allocated":5201920,"data_stored":3172675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":810282323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-18,"num_object_clones":0,"num_object_copies":-36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-18,"num_whiteouts":0,"num_read":-7975,"num_read_kb":-14122,"num_write":-971,"num_write_kb":-2372,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.941017"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.368184+0000","last_change":"2026-02-22T21:45:47.055786+0000","last_active":"2026-02-22T22:09:41.368184+0000","last_peered":"2026-02-22T22:09:41.368184+0000","last_clean":"2026-02-22T22:09:41.368184+0000","last_became_active":"2026-02-22T21:45:45.053797+0000","last_became_peered":"2026-02-22T21:45:45.053797+0000","last_unstale":"2026-02-22T22:09:41.368184+0000","last_undegraded":"2026-02-22T22:09:41.368184+0000","last_fullsized":"2026-02-22T22:09:41.368184+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:53:17.577669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050143900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370673+0000","last_change":"2026-02-22T21:45:47.055663+0000","last_active":"2026-02-22T22:09:41.370673+0000","last_peered":"2026-02-22T22:09:41.370673+0000","last_clean":"2026-02-22T22:09:41.370673+0000","last_became_active":"2026-02-22T21:45:45.053799+0000","last_became_peered":"2026-02-22T21:45:45.053799+0000","last_unstale":"2026-02-22T22:09:41.370673+0000","last_undegraded":"2026-02-22T22:09:41.370673+0000","last_fullsized":"2026-02-22T22:09:41.370673+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:27:16.854273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0003992,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370478+0000","last_change":"2026-02-22T21:45:47.055898+0000","last_active":"2026-02-22T22:09:41.370478+0000","last_peered":"2026-02-22T22:09:41.370478+0000","last_clean":"2026-02-22T22:09:41.370478+0000","last_became_active":"2026-02-22T21:45:45.053506+0000","last_became_peered":"2026-02-22T21:45:45.053506+0000","last_unstale":"2026-02-22T22:09:41.370478+0000","last_undegraded":"2026-02-22T22:09:41.370478+0000","last_fullsized":"2026-02-22T22:09:41.370478+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:10:14.553071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060315800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.367026+0000","last_change":"2026-02-22T21:45:47.055731+0000","last_active":"2026-02-22T22:09:41.367026+0000","last_peered":"2026-02-22T22:09:41.367026+0000","last_clean":"2026-02-22T22:09:41.367026+0000","last_became_active":"2026-02-22T21:45:45.053788+0000","last_became_peered":"2026-02-22T21:45:45.053788+0000","last_unstale":"2026-02-22T22:09:41.367026+0000","last_undegraded":"2026-02-22T22:09:41.367026+0000","last_fullsized":"2026-02-22T22:09:41.367026+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:15:31.306502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048115199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":312,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.456266+0000","last_change":"2026-02-22T21:45:47.057685+0000","last_active":"2026-02-22T22:09:41.456266+0000","last_peered":"2026-02-22T22:09:41.456266+0000","last_clean":"2026-02-22T22:09:41.456266+0000","last_became_active":"2026-02-22T21:45:45.053848+0000","last_became_peered":"2026-02-22T21:45:45.053848+0000","last_unstale":"2026-02-22T22:09:41.456266+0000","last_undegraded":"2026-02-22T22:09:41.456266+0000","last_fullsized":"2026-02-22T22:09:41.456266+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:30:42.167683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089195400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":308,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.457119+0000","last_change":"2026-02-22T21:45:47.106335+0000","last_active":"2026-02-22T22:09:41.457119+0000","last_peered":"2026-02-22T22:09:41.457119+0000","last_clean":"2026-02-22T22:09:41.457119+0000","last_became_active":"2026-02-22T21:45:45.053963+0000","last_became_peered":"2026-02-22T21:45:45.053963+0000","last_unstale":"2026-02-22T22:09:41.457119+0000","last_undegraded":"2026-02-22T22:09:41.457119+0000","last_fullsized":"2026-02-22T22:09:41.457119+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:40:42.243266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000224897,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.455250+0000","last_change":"2026-02-22T21:45:47.106341+0000","last_active":"2026-02-22T22:09:41.455250+0000","last_peered":"2026-02-22T22:09:41.455250+0000","last_clean":"2026-02-22T22:09:41.455250+0000","last_became_active":"2026-02-22T21:45:45.053834+0000","last_became_peered":"2026-02-22T21:45:45.053834+0000","last_unstale":"2026-02-22T22:09:41.455250+0000","last_undegraded":"2026-02-22T22:09:41.455250+0000","last_fullsized":"2026-02-22T22:09:41.455250+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:44:15.059855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000242379,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":310,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.369837+0000","last_change":"2026-02-22T21:45:47.056034+0000","last_active":"2026-02-22T22:09:41.369837+0000","last_peered":"2026-02-22T22:09:41.369837+0000","last_clean":"2026-02-22T22:09:41.369837+0000","last_became_active":"2026-02-22T21:45:45.053434+0000","last_became_peered":"2026-02-22T21:45:45.053434+0000","last_unstale":"2026-02-22T22:09:41.369837+0000","last_undegraded":"2026-02-22T22:09:41.369837+0000","last_fullsized":"2026-02-22T22:09:41.369837+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:45.436732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076235300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":400,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.364574+0000","last_change":"2026-02-22T21:45:41.044949+0000","last_active":"2026-02-22T22:09:41.364574+0000","last_peered":"2026-02-22T22:09:41.364574+0000","last_clean":"2026-02-22T22:09:41.364574+0000","last_became_active":"2026-02-22T21:45:41.044711+0000","last_became_peered":"2026-02-22T21:45:41.044711+0000","last_unstale":"2026-02-22T22:09:41.364574+0000","last_undegraded":"2026-02-22T22:09:41.364574+0000","last_fullsized":"2026-02-22T22:09:41.364574+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_clean_scrub_stamp":"2026-02-22T21:45:40.040843+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:25:57.694568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705956,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":239212,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":237886,"kb_avail":94132628,"statfs":{"total":96636764160,"available":96391811072,"internally_reserved":0,"allocated":1335296,"data_stored":663967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243595719},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705958,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":285556,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":283646,"kb_avail":94086284,"statfs":{"total":96636764160,"available":96344354816,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290453958},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738663,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":271668,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":269758,"kb_avail":94100172,"statfs":{"total":96636764160,"available":96358576128,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":276232646},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:09:50.231 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:09:50.388 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T22:09:50.388 INFO:teuthology.orchestra.run.trial104.stdout:{"epoch":156,"fsid":"e1c6abd2-8277-4fd0-b513-f90ef487dc18","created":"2026-02-22T21:45:29.235879+0000","modified":"2026-02-22T22:09:41.360325+0000","last_up_change":"2026-02-22T21:45:39.040408+0000","last_in_change":"2026-02-22T21:45:29.793510+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:45:39.306938+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:45:43.402955+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"41ca61ad-dac2-4161-b228-821056447285","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6808","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6809","nonce":895813024}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6810","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6811","nonce":895813024}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6814","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6815","nonce":895813024}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6812","nonce":895813024},{"type":"v1","addr":"10.20.193.104:6813","nonce":895813024}]},"public_addr":"10.20.193.104:6809/895813024","cluster_addr":"10.20.193.104:6811/895813024","heartbeat_back_addr":"10.20.193.104:6815/895813024","heartbeat_front_addr":"10.20.193.104:6813/895813024","state":["exists","up"]},{"osd":1,"uuid":"3f8ea715-66fc-455f-b443-a336ef783866","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6800","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6801","nonce":251451549}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6802","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6803","nonce":251451549}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6806","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6807","nonce":251451549}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6804","nonce":251451549},{"type":"v1","addr":"10.20.193.104:6805","nonce":251451549}]},"public_addr":"10.20.193.104:6801/251451549","cluster_addr":"10.20.193.104:6803/251451549","heartbeat_back_addr":"10.20.193.104:6807/251451549","heartbeat_front_addr":"10.20.193.104:6805/251451549","state":["exists","up"]},{"osd":2,"uuid":"ee29ada6-f1b6-41d9-aa45-93db0f1af5c4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":96,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6816","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6817","nonce":2738971318}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6818","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6819","nonce":2738971318}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6822","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6823","nonce":2738971318}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.104:6820","nonce":2738971318},{"type":"v1","addr":"10.20.193.104:6821","nonce":2738971318}]},"public_addr":"10.20.193.104:6817/2738971318","cluster_addr":"10.20.193.104:6819/2738971318","heartbeat_back_addr":"10.20.193.104:6823/2738971318","heartbeat_front_addr":"10.20.193.104:6821/2738971318","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.197574+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:36.954361+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:45:37.258932+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":41,"length":2}]},{"pool":5,"snaps":[{"begin":4,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:09:51.397 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-22T22:09:51.397 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:09:51.477 INFO:teuthology.orchestra.run.trial104.stdout:{ 2026-02-22T22:09:51.478 INFO:teuthology.orchestra.run.trial104.stdout: "success": "" 2026-02-22T22:09:51.478 INFO:teuthology.orchestra.run.trial104.stdout:} 2026-02-22T22:09:51.484 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-22T22:09:51.642 INFO:teuthology.orchestra.run.trial104.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:09:51.652 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-22T22:09:51.652 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:09:51.733 INFO:teuthology.orchestra.run.trial104.stdout:{ 2026-02-22T22:09:51.734 INFO:teuthology.orchestra.run.trial104.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T22:09:51.734 INFO:teuthology.orchestra.run.trial104.stdout:} 2026-02-22T22:09:51.739 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-22T22:09:51.897 INFO:teuthology.orchestra.run.trial104.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:09:51.906 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-22T22:09:51.906 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:09:51.985 INFO:teuthology.orchestra.run.trial104.stdout:{ 2026-02-22T22:09:51.985 INFO:teuthology.orchestra.run.trial104.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) " 2026-02-22T22:09:51.985 INFO:teuthology.orchestra.run.trial104.stdout:} 2026-02-22T22:09:51.992 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-22T22:09:52.148 INFO:teuthology.orchestra.run.trial104.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:09:52.157 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:09:52.316 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T22:09:52.317 INFO:teuthology.orchestra.run.trial104.stderr:dumped all 2026-02-22T22:09:52.325 INFO:teuthology.orchestra.run.trial104.stdout:{"pg_ready":true,"pg_map":{"version":881,"stamp":"2026-02-22T22:09:51.301360+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":796436,"kb_used_data":5080,"kb_used_omap":4,"kb_used_meta":791291,"kb_avail":282319084,"statfs":{"total":289910292480,"available":289094742016,"internally_reserved":0,"allocated":5201920,"data_stored":3172675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":810282323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-19,"num_objects":-18,"num_object_clones":0,"num_object_copies":-36,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-18,"num_whiteouts":0,"num_read":-7979,"num_read_kb":-14126,"num_write":-971,"num_write_kb":-2372,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-8,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.001261"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.368184+0000","last_change":"2026-02-22T21:45:47.055786+0000","last_active":"2026-02-22T22:09:41.368184+0000","last_peered":"2026-02-22T22:09:41.368184+0000","last_clean":"2026-02-22T22:09:41.368184+0000","last_became_active":"2026-02-22T21:45:45.053797+0000","last_became_peered":"2026-02-22T21:45:45.053797+0000","last_unstale":"2026-02-22T22:09:41.368184+0000","last_undegraded":"2026-02-22T22:09:41.368184+0000","last_fullsized":"2026-02-22T22:09:41.368184+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:53:17.577669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050143900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370673+0000","last_change":"2026-02-22T21:45:47.055663+0000","last_active":"2026-02-22T22:09:41.370673+0000","last_peered":"2026-02-22T22:09:41.370673+0000","last_clean":"2026-02-22T22:09:41.370673+0000","last_became_active":"2026-02-22T21:45:45.053799+0000","last_became_peered":"2026-02-22T21:45:45.053799+0000","last_unstale":"2026-02-22T22:09:41.370673+0000","last_undegraded":"2026-02-22T22:09:41.370673+0000","last_fullsized":"2026-02-22T22:09:41.370673+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:27:16.854273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0003992,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.370478+0000","last_change":"2026-02-22T21:45:47.055898+0000","last_active":"2026-02-22T22:09:41.370478+0000","last_peered":"2026-02-22T22:09:41.370478+0000","last_clean":"2026-02-22T22:09:41.370478+0000","last_became_active":"2026-02-22T21:45:45.053506+0000","last_became_peered":"2026-02-22T21:45:45.053506+0000","last_unstale":"2026-02-22T22:09:41.370478+0000","last_undegraded":"2026-02-22T22:09:41.370478+0000","last_fullsized":"2026-02-22T22:09:41.370478+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:10:14.553071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060315800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.367026+0000","last_change":"2026-02-22T21:45:47.055731+0000","last_active":"2026-02-22T22:09:41.367026+0000","last_peered":"2026-02-22T22:09:41.367026+0000","last_clean":"2026-02-22T22:09:41.367026+0000","last_became_active":"2026-02-22T21:45:45.053788+0000","last_became_peered":"2026-02-22T21:45:45.053788+0000","last_unstale":"2026-02-22T22:09:41.367026+0000","last_undegraded":"2026-02-22T22:09:41.367026+0000","last_fullsized":"2026-02-22T22:09:41.367026+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:15:31.306502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048115199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":312,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.456266+0000","last_change":"2026-02-22T21:45:47.057685+0000","last_active":"2026-02-22T22:09:41.456266+0000","last_peered":"2026-02-22T22:09:41.456266+0000","last_clean":"2026-02-22T22:09:41.456266+0000","last_became_active":"2026-02-22T21:45:45.053848+0000","last_became_peered":"2026-02-22T21:45:45.053848+0000","last_unstale":"2026-02-22T22:09:41.456266+0000","last_undegraded":"2026-02-22T22:09:41.456266+0000","last_fullsized":"2026-02-22T22:09:41.456266+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:30:42.167683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089195400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":308,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.457119+0000","last_change":"2026-02-22T21:45:47.106335+0000","last_active":"2026-02-22T22:09:41.457119+0000","last_peered":"2026-02-22T22:09:41.457119+0000","last_clean":"2026-02-22T22:09:41.457119+0000","last_became_active":"2026-02-22T21:45:45.053963+0000","last_became_peered":"2026-02-22T21:45:45.053963+0000","last_unstale":"2026-02-22T22:09:41.457119+0000","last_undegraded":"2026-02-22T22:09:41.457119+0000","last_fullsized":"2026-02-22T22:09:41.457119+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:40:42.243266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000224897,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":309,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.455250+0000","last_change":"2026-02-22T21:45:47.106341+0000","last_active":"2026-02-22T22:09:41.455250+0000","last_peered":"2026-02-22T22:09:41.455250+0000","last_clean":"2026-02-22T22:09:41.455250+0000","last_became_active":"2026-02-22T21:45:45.053834+0000","last_became_peered":"2026-02-22T21:45:45.053834+0000","last_unstale":"2026-02-22T22:09:41.455250+0000","last_undegraded":"2026-02-22T22:09:41.455250+0000","last_fullsized":"2026-02-22T22:09:41.455250+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:44:15.059855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000242379,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":310,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.369837+0000","last_change":"2026-02-22T21:45:47.056034+0000","last_active":"2026-02-22T22:09:41.369837+0000","last_peered":"2026-02-22T22:09:41.369837+0000","last_clean":"2026-02-22T22:09:41.369837+0000","last_became_active":"2026-02-22T21:45:45.053434+0000","last_became_peered":"2026-02-22T21:45:45.053434+0000","last_unstale":"2026-02-22T22:09:41.369837+0000","last_undegraded":"2026-02-22T22:09:41.369837+0000","last_fullsized":"2026-02-22T22:09:41.369837+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:44.047022+0000","last_clean_scrub_stamp":"2026-02-22T21:45:44.047022+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:45.436732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076235300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":400,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.364574+0000","last_change":"2026-02-22T21:45:41.044949+0000","last_active":"2026-02-22T22:09:41.364574+0000","last_peered":"2026-02-22T22:09:41.364574+0000","last_clean":"2026-02-22T22:09:41.364574+0000","last_became_active":"2026-02-22T21:45:41.044711+0000","last_became_peered":"2026-02-22T21:45:41.044711+0000","last_unstale":"2026-02-22T22:09:41.364574+0000","last_undegraded":"2026-02-22T22:09:41.364574+0000","last_fullsized":"2026-02-22T22:09:41.364574+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:40.040843+0000","last_clean_scrub_stamp":"2026-02-22T21:45:40.040843+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:25:57.694568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705957,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":239212,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":237886,"kb_avail":94132628,"statfs":{"total":96636764160,"available":96391811072,"internally_reserved":0,"allocated":1335296,"data_stored":663967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243595719},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705959,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":285556,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":283646,"kb_avail":94086284,"statfs":{"total":96636764160,"available":96344354816,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290453958},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738663,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":271668,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":269758,"kb_avail":94100172,"statfs":{"total":96636764160,"available":96358576128,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":276232646},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:09:52.332 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.333 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.333 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.334 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.334 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.334 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.334 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.335 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T21:45:44.047022+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=44, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.335 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T21:45:40.040843+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=40, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=9, tm_sec=50, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:09:52.335 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:10:12.337 DEBUG:teuthology.orchestra.run.trial104:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:10:12.505 INFO:teuthology.orchestra.run.trial104.stdout: 2026-02-22T22:10:12.506 INFO:teuthology.orchestra.run.trial104.stderr:dumped all 2026-02-22T22:10:12.513 INFO:teuthology.orchestra.run.trial104.stdout:{"pg_ready":true,"pg_map":{"version":891,"stamp":"2026-02-22T22:10:11.302980+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":796424,"kb_used_data":5080,"kb_used_omap":4,"kb_used_meta":791291,"kb_avail":282319096,"statfs":{"total":289910292480,"available":289094754304,"internally_reserved":0,"allocated":5201920,"data_stored":3172675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4781,"internal_metadata":810282323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000937"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":318,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:57.396061+0000","last_change":"2026-02-22T22:09:57.396072+0000","last_active":"2026-02-22T22:09:57.396061+0000","last_peered":"2026-02-22T22:09:57.396061+0000","last_clean":"2026-02-22T22:09:57.396061+0000","last_became_active":"2026-02-22T21:45:45.053797+0000","last_became_peered":"2026-02-22T21:45:45.053797+0000","last_unstale":"2026-02-22T22:09:57.396061+0000","last_undegraded":"2026-02-22T22:09:57.396061+0000","last_fullsized":"2026-02-22T22:09:57.396061+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:09:57.396023+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:09:57.396023+0000","last_clean_scrub_stamp":"2026-02-22T22:09:57.396023+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:41:15.607597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050143900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":318,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:56.412455+0000","last_change":"2026-02-22T22:09:56.412474+0000","last_active":"2026-02-22T22:09:56.412455+0000","last_peered":"2026-02-22T22:09:56.412455+0000","last_clean":"2026-02-22T22:09:56.412455+0000","last_became_active":"2026-02-22T21:45:45.053799+0000","last_became_peered":"2026-02-22T21:45:45.053799+0000","last_unstale":"2026-02-22T22:09:56.412455+0000","last_undegraded":"2026-02-22T22:09:56.412455+0000","last_fullsized":"2026-02-22T22:09:56.412455+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:09:56.412376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:09:56.412376+0000","last_clean_scrub_stamp":"2026-02-22T22:09:56.412376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:17:59.107361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0003992,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":318,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:53.344461+0000","last_change":"2026-02-22T22:09:53.344477+0000","last_active":"2026-02-22T22:09:53.344461+0000","last_peered":"2026-02-22T22:09:53.344461+0000","last_clean":"2026-02-22T22:09:53.344461+0000","last_became_active":"2026-02-22T21:45:45.053506+0000","last_became_peered":"2026-02-22T21:45:45.053506+0000","last_unstale":"2026-02-22T22:09:53.344461+0000","last_undegraded":"2026-02-22T22:09:53.344461+0000","last_fullsized":"2026-02-22T22:09:53.344461+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:09:53.344398+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:09:53.344398+0000","last_clean_scrub_stamp":"2026-02-22T22:09:53.344398+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:01:58.402246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060315800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":318,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:54.367419+0000","last_change":"2026-02-22T22:09:54.367428+0000","last_active":"2026-02-22T22:09:54.367419+0000","last_peered":"2026-02-22T22:09:54.367419+0000","last_clean":"2026-02-22T22:09:54.367419+0000","last_became_active":"2026-02-22T21:45:45.053788+0000","last_became_peered":"2026-02-22T21:45:45.053788+0000","last_unstale":"2026-02-22T22:09:54.367419+0000","last_undegraded":"2026-02-22T22:09:54.367419+0000","last_fullsized":"2026-02-22T22:09:54.367419+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:09:54.367369+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:09:54.367369+0000","last_clean_scrub_stamp":"2026-02-22T22:09:54.367369+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:58:36.513255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048115199999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"156'3","reported_seq":322,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:51.868670+0000","last_change":"2026-02-22T22:09:51.868685+0000","last_active":"2026-02-22T22:09:51.868670+0000","last_peered":"2026-02-22T22:09:51.868670+0000","last_clean":"2026-02-22T22:09:51.868670+0000","last_became_active":"2026-02-22T21:45:45.053848+0000","last_became_peered":"2026-02-22T21:45:45.053848+0000","last_unstale":"2026-02-22T22:09:51.868670+0000","last_undegraded":"2026-02-22T22:09:51.868670+0000","last_fullsized":"2026-02-22T22:09:51.868670+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"156'3","last_scrub_stamp":"2026-02-22T22:09:51.868635+0000","last_deep_scrub":"156'3","last_deep_scrub_stamp":"2026-02-22T22:09:51.868635+0000","last_clean_scrub_stamp":"2026-02-22T22:09:51.868635+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:05:24.540347+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00089195400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":317,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:53.240320+0000","last_change":"2026-02-22T22:09:53.240330+0000","last_active":"2026-02-22T22:09:53.240320+0000","last_peered":"2026-02-22T22:09:53.240320+0000","last_clean":"2026-02-22T22:09:53.240320+0000","last_became_active":"2026-02-22T21:45:45.053963+0000","last_became_peered":"2026-02-22T21:45:45.053963+0000","last_unstale":"2026-02-22T22:09:53.240320+0000","last_undegraded":"2026-02-22T22:09:53.240320+0000","last_fullsized":"2026-02-22T22:09:53.240320+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:09:53.240275+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:09:53.240275+0000","last_clean_scrub_stamp":"2026-02-22T22:09:53.240275+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:46:10.503396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000224897,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":318,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:52.251124+0000","last_change":"2026-02-22T22:09:52.251137+0000","last_active":"2026-02-22T22:09:52.251124+0000","last_peered":"2026-02-22T22:09:52.251124+0000","last_clean":"2026-02-22T22:09:52.251124+0000","last_became_active":"2026-02-22T21:45:45.053834+0000","last_became_peered":"2026-02-22T21:45:45.053834+0000","last_unstale":"2026-02-22T22:09:52.251124+0000","last_undegraded":"2026-02-22T22:09:52.251124+0000","last_fullsized":"2026-02-22T22:09:52.251124+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:09:52.251067+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:09:52.251067+0000","last_clean_scrub_stamp":"2026-02-22T22:09:52.251067+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:58:39.093870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000242379,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"156'2","reported_seq":320,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:55.384888+0000","last_change":"2026-02-22T22:09:55.384905+0000","last_active":"2026-02-22T22:09:55.384888+0000","last_peered":"2026-02-22T22:09:55.384888+0000","last_clean":"2026-02-22T22:09:55.384888+0000","last_became_active":"2026-02-22T21:45:45.053434+0000","last_became_peered":"2026-02-22T21:45:45.053434+0000","last_unstale":"2026-02-22T22:09:55.384888+0000","last_undegraded":"2026-02-22T22:09:55.384888+0000","last_fullsized":"2026-02-22T22:09:55.384888+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"156'2","last_scrub_stamp":"2026-02-22T22:09:55.384848+0000","last_deep_scrub":"156'2","last_deep_scrub_stamp":"2026-02-22T22:09:55.384848+0000","last_clean_scrub_stamp":"2026-02-22T22:09:55.384848+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:34:41.588424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00076235300000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"156'74","reported_seq":411,"reported_epoch":156,"state":"active+clean","last_fresh":"2026-02-22T22:09:52.382698+0000","last_change":"2026-02-22T22:09:52.382718+0000","last_active":"2026-02-22T22:09:52.382698+0000","last_peered":"2026-02-22T22:09:52.382698+0000","last_clean":"2026-02-22T22:09:52.382698+0000","last_became_active":"2026-02-22T21:45:41.044711+0000","last_became_peered":"2026-02-22T21:45:41.044711+0000","last_unstale":"2026-02-22T22:09:52.382698+0000","last_undegraded":"2026-02-22T22:09:52.382698+0000","last_fullsized":"2026-02-22T22:09:52.382698+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"156'74","last_scrub_stamp":"2026-02-22T22:09:52.382651+0000","last_deep_scrub":"156'74","last_deep_scrub_stamp":"2026-02-22T22:09:52.382651+0000","last_clean_scrub_stamp":"2026-02-22T22:09:52.382651+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:02:00.137697+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705961,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":239208,"kb_used_data":1304,"kb_used_omap":1,"kb_used_meta":237886,"kb_avail":94132632,"statfs":{"total":96636764160,"available":96391815168,"internally_reserved":0,"allocated":1335296,"data_stored":663967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":243595719},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":9,"seq":38654705963,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":285552,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":283646,"kb_avail":94086288,"statfs":{"total":96636764160,"available":96344358912,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290453958},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738667,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":271664,"kb_used_data":1888,"kb_used_omap":1,"kb_used_meta":269758,"kb_avail":94100176,"statfs":{"total":96636764160,"available":96358580224,"internally_reserved":0,"allocated":1933312,"data_stored":1254354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":276232646},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:10:12.515 DEBUG:teuthology.orchestra.run.trial104:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-22T22:10:12.682 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:10:12.683 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:10:12.683 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-22T22:10:12.683 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:12.732 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-22T22:10:12.732 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-22T22:10:12.732 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:12.777 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-22T22:10:12.777 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-22T22:10:12.778 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:12.825 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-22T22:10:12.825 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:10:12.826 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-22T22:10:12.826 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:12.871 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-22T22:10:12.871 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:10:12.871 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-22T22:10:12.871 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:12.881 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-22T22:10:12.882 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:10:12.882 DEBUG:teuthology.orchestra.run.trial104:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:10:12.948 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial104.front.sepia.ceph.com 2026-02-22T22:10:12.948 DEBUG:teuthology.orchestra.run.trial104:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-22T22:10:13.030 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial104.front.sepia.ceph.com 2026-02-22T22:10:13.030 DEBUG:teuthology.orchestra.run.trial104:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-22T22:10:13.108 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial104.front.sepia.ceph.com 2026-02-22T22:10:13.108 DEBUG:teuthology.orchestra.run.trial104:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-22T22:10:13.192 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:10:13.194 DEBUG:teuthology.misc:Transferring archived files from trial104:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/data/cluster2.mon.a.tgz 2026-02-22T22:10:13.194 DEBUG:teuthology.orchestra.run.trial104:> mktemp 2026-02-22T22:10:13.210 INFO:teuthology.orchestra.run.trial104.stdout:/tmp/tmp.k6TlqZWh1c 2026-02-22T22:10:13.211 DEBUG:teuthology.orchestra.run.trial104:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.k6TlqZWh1c 2026-02-22T22:10:13.327 DEBUG:teuthology.orchestra.run.trial104:> sudo chmod 0666 /tmp/tmp.k6TlqZWh1c 2026-02-22T22:10:13.407 DEBUG:teuthology.orchestra.remote:trial104:/tmp/tmp.k6TlqZWh1c is 431KB 2026-02-22T22:10:13.503 DEBUG:teuthology.orchestra.run.trial104:> rm -fr /tmp/tmp.k6TlqZWh1c 2026-02-22T22:10:13.521 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:10:13.521 DEBUG:teuthology.orchestra.run.trial048:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:10:13.524 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:10:13.584 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:10:13.599 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:10:13.599 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:10:13.761 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T22:10:13.761 INFO:teuthology.orchestra.run.trial048.stderr:dumped all 2026-02-22T22:10:13.770 INFO:teuthology.orchestra.run.trial048.stdout:{"pg_ready":true,"pg_map":{"version":923,"stamp":"2026-02-22T22:10:12.744298+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":467816,"kb_used_data":5356,"kb_used_omap":4,"kb_used_meta":462395,"kb_avail":282647704,"statfs":{"total":289910292480,"available":289431248896,"internally_reserved":0,"allocated":5484544,"data_stored":3320314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":473492818},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000912"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660753+0000","last_change":"2026-02-22T21:45:06.268535+0000","last_active":"2026-02-22T22:09:41.660753+0000","last_peered":"2026-02-22T22:09:41.660753+0000","last_clean":"2026-02-22T22:09:41.660753+0000","last_became_active":"2026-02-22T21:45:04.266643+0000","last_became_peered":"2026-02-22T21:45:04.266643+0000","last_unstale":"2026-02-22T22:09:41.660753+0000","last_undegraded":"2026-02-22T22:09:41.660753+0000","last_fullsized":"2026-02-22T22:09:41.660753+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:35.211466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082153,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661288+0000","last_change":"2026-02-22T21:45:06.268006+0000","last_active":"2026-02-22T22:09:41.661288+0000","last_peered":"2026-02-22T22:09:41.661288+0000","last_clean":"2026-02-22T22:09:41.661288+0000","last_became_active":"2026-02-22T21:45:04.266954+0000","last_became_peered":"2026-02-22T21:45:04.266954+0000","last_unstale":"2026-02-22T22:09:41.661288+0000","last_undegraded":"2026-02-22T22:09:41.661288+0000","last_fullsized":"2026-02-22T22:09:41.661288+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:27:29.460040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030903099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.658947+0000","last_change":"2026-02-22T21:45:06.268671+0000","last_active":"2026-02-22T22:09:41.658947+0000","last_peered":"2026-02-22T22:09:41.658947+0000","last_clean":"2026-02-22T22:09:41.658947+0000","last_became_active":"2026-02-22T21:45:04.266721+0000","last_became_peered":"2026-02-22T21:45:04.266721+0000","last_unstale":"2026-02-22T22:09:41.658947+0000","last_undegraded":"2026-02-22T22:09:41.658947+0000","last_fullsized":"2026-02-22T22:09:41.658947+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:11:11.390548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094190300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660725+0000","last_change":"2026-02-22T21:45:06.268222+0000","last_active":"2026-02-22T22:09:41.660725+0000","last_peered":"2026-02-22T22:09:41.660725+0000","last_clean":"2026-02-22T22:09:41.660725+0000","last_became_active":"2026-02-22T21:45:04.266897+0000","last_became_peered":"2026-02-22T21:45:04.266897+0000","last_unstale":"2026-02-22T22:09:41.660725+0000","last_undegraded":"2026-02-22T22:09:41.660725+0000","last_fullsized":"2026-02-22T22:09:41.660725+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:10.132872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000509874,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:40.349127+0000","last_change":"2026-02-22T21:45:06.269931+0000","last_active":"2026-02-22T22:09:40.349127+0000","last_peered":"2026-02-22T22:09:40.349127+0000","last_clean":"2026-02-22T22:09:40.349127+0000","last_became_active":"2026-02-22T21:45:04.266353+0000","last_became_peered":"2026-02-22T21:45:04.266353+0000","last_unstale":"2026-02-22T22:09:40.349127+0000","last_undegraded":"2026-02-22T22:09:40.349127+0000","last_fullsized":"2026-02-22T22:09:40.349127+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:48:09.631675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080603100000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.539967+0000","last_change":"2026-02-22T21:45:06.795048+0000","last_active":"2026-02-22T22:09:41.539967+0000","last_peered":"2026-02-22T22:09:41.539967+0000","last_clean":"2026-02-22T22:09:41.539967+0000","last_became_active":"2026-02-22T21:45:04.266437+0000","last_became_peered":"2026-02-22T21:45:04.266437+0000","last_unstale":"2026-02-22T22:09:41.539967+0000","last_undegraded":"2026-02-22T22:09:41.539967+0000","last_fullsized":"2026-02-22T22:09:41.539967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:12:37.999670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021661099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.536537+0000","last_change":"2026-02-22T21:45:06.795071+0000","last_active":"2026-02-22T22:09:41.536537+0000","last_peered":"2026-02-22T22:09:41.536537+0000","last_clean":"2026-02-22T22:09:41.536537+0000","last_became_active":"2026-02-22T21:45:04.266381+0000","last_became_peered":"2026-02-22T21:45:04.266381+0000","last_unstale":"2026-02-22T22:09:41.536537+0000","last_undegraded":"2026-02-22T22:09:41.536537+0000","last_fullsized":"2026-02-22T22:09:41.536537+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:59.877077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022258099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661093+0000","last_change":"2026-02-22T21:45:06.268349+0000","last_active":"2026-02-22T22:09:41.661093+0000","last_peered":"2026-02-22T22:09:41.661093+0000","last_clean":"2026-02-22T22:09:41.661093+0000","last_became_active":"2026-02-22T21:45:04.266127+0000","last_became_peered":"2026-02-22T21:45:04.266127+0000","last_unstale":"2026-02-22T22:09:41.661093+0000","last_undegraded":"2026-02-22T22:09:41.661093+0000","last_fullsized":"2026-02-22T22:09:41.661093+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:39:02.311582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065919899999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":424,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.658276+0000","last_change":"2026-02-22T21:45:01.134350+0000","last_active":"2026-02-22T22:09:41.658276+0000","last_peered":"2026-02-22T22:09:41.658276+0000","last_clean":"2026-02-22T22:09:41.658276+0000","last_became_active":"2026-02-22T21:45:01.133900+0000","last_became_peered":"2026-02-22T21:45:01.133900+0000","last_unstale":"2026-02-22T22:09:41.658276+0000","last_undegraded":"2026-02-22T22:09:41.658276+0000","last_fullsized":"2026-02-22T22:09:41.658276+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_clean_scrub_stamp":"2026-02-22T21:44:59.253183+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:32:00.074720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705968,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":144904,"kb_used_data":1396,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":94226936,"statfs":{"total":96636764160,"available":96488382464,"internally_reserved":0,"allocated":1429504,"data_stored":713180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738673,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":159376,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":94212464,"statfs":{"total":96636764160,"available":96473563136,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161151430},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738675,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":163536,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":94208304,"statfs":{"total":96636764160,"available":96469303296,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:10:13.772 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:10:13.932 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T22:10:13.932 INFO:teuthology.orchestra.run.trial048.stderr:dumped all 2026-02-22T22:10:13.940 INFO:teuthology.orchestra.run.trial048.stdout:{"pg_ready":true,"pg_map":{"version":923,"stamp":"2026-02-22T22:10:12.744298+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":467816,"kb_used_data":5356,"kb_used_omap":4,"kb_used_meta":462395,"kb_avail":282647704,"statfs":{"total":289910292480,"available":289431248896,"internally_reserved":0,"allocated":5484544,"data_stored":3320314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":473492818},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000912"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660753+0000","last_change":"2026-02-22T21:45:06.268535+0000","last_active":"2026-02-22T22:09:41.660753+0000","last_peered":"2026-02-22T22:09:41.660753+0000","last_clean":"2026-02-22T22:09:41.660753+0000","last_became_active":"2026-02-22T21:45:04.266643+0000","last_became_peered":"2026-02-22T21:45:04.266643+0000","last_unstale":"2026-02-22T22:09:41.660753+0000","last_undegraded":"2026-02-22T22:09:41.660753+0000","last_fullsized":"2026-02-22T22:09:41.660753+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:35.211466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082153,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661288+0000","last_change":"2026-02-22T21:45:06.268006+0000","last_active":"2026-02-22T22:09:41.661288+0000","last_peered":"2026-02-22T22:09:41.661288+0000","last_clean":"2026-02-22T22:09:41.661288+0000","last_became_active":"2026-02-22T21:45:04.266954+0000","last_became_peered":"2026-02-22T21:45:04.266954+0000","last_unstale":"2026-02-22T22:09:41.661288+0000","last_undegraded":"2026-02-22T22:09:41.661288+0000","last_fullsized":"2026-02-22T22:09:41.661288+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:27:29.460040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030903099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.658947+0000","last_change":"2026-02-22T21:45:06.268671+0000","last_active":"2026-02-22T22:09:41.658947+0000","last_peered":"2026-02-22T22:09:41.658947+0000","last_clean":"2026-02-22T22:09:41.658947+0000","last_became_active":"2026-02-22T21:45:04.266721+0000","last_became_peered":"2026-02-22T21:45:04.266721+0000","last_unstale":"2026-02-22T22:09:41.658947+0000","last_undegraded":"2026-02-22T22:09:41.658947+0000","last_fullsized":"2026-02-22T22:09:41.658947+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:11:11.390548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094190300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660725+0000","last_change":"2026-02-22T21:45:06.268222+0000","last_active":"2026-02-22T22:09:41.660725+0000","last_peered":"2026-02-22T22:09:41.660725+0000","last_clean":"2026-02-22T22:09:41.660725+0000","last_became_active":"2026-02-22T21:45:04.266897+0000","last_became_peered":"2026-02-22T21:45:04.266897+0000","last_unstale":"2026-02-22T22:09:41.660725+0000","last_undegraded":"2026-02-22T22:09:41.660725+0000","last_fullsized":"2026-02-22T22:09:41.660725+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:10.132872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000509874,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:40.349127+0000","last_change":"2026-02-22T21:45:06.269931+0000","last_active":"2026-02-22T22:09:40.349127+0000","last_peered":"2026-02-22T22:09:40.349127+0000","last_clean":"2026-02-22T22:09:40.349127+0000","last_became_active":"2026-02-22T21:45:04.266353+0000","last_became_peered":"2026-02-22T21:45:04.266353+0000","last_unstale":"2026-02-22T22:09:40.349127+0000","last_undegraded":"2026-02-22T22:09:40.349127+0000","last_fullsized":"2026-02-22T22:09:40.349127+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:48:09.631675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080603100000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.539967+0000","last_change":"2026-02-22T21:45:06.795048+0000","last_active":"2026-02-22T22:09:41.539967+0000","last_peered":"2026-02-22T22:09:41.539967+0000","last_clean":"2026-02-22T22:09:41.539967+0000","last_became_active":"2026-02-22T21:45:04.266437+0000","last_became_peered":"2026-02-22T21:45:04.266437+0000","last_unstale":"2026-02-22T22:09:41.539967+0000","last_undegraded":"2026-02-22T22:09:41.539967+0000","last_fullsized":"2026-02-22T22:09:41.539967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:12:37.999670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021661099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.536537+0000","last_change":"2026-02-22T21:45:06.795071+0000","last_active":"2026-02-22T22:09:41.536537+0000","last_peered":"2026-02-22T22:09:41.536537+0000","last_clean":"2026-02-22T22:09:41.536537+0000","last_became_active":"2026-02-22T21:45:04.266381+0000","last_became_peered":"2026-02-22T21:45:04.266381+0000","last_unstale":"2026-02-22T22:09:41.536537+0000","last_undegraded":"2026-02-22T22:09:41.536537+0000","last_fullsized":"2026-02-22T22:09:41.536537+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:59.877077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022258099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661093+0000","last_change":"2026-02-22T21:45:06.268349+0000","last_active":"2026-02-22T22:09:41.661093+0000","last_peered":"2026-02-22T22:09:41.661093+0000","last_clean":"2026-02-22T22:09:41.661093+0000","last_became_active":"2026-02-22T21:45:04.266127+0000","last_became_peered":"2026-02-22T21:45:04.266127+0000","last_unstale":"2026-02-22T22:09:41.661093+0000","last_undegraded":"2026-02-22T22:09:41.661093+0000","last_fullsized":"2026-02-22T22:09:41.661093+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:39:02.311582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065919899999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":424,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.658276+0000","last_change":"2026-02-22T21:45:01.134350+0000","last_active":"2026-02-22T22:09:41.658276+0000","last_peered":"2026-02-22T22:09:41.658276+0000","last_clean":"2026-02-22T22:09:41.658276+0000","last_became_active":"2026-02-22T21:45:01.133900+0000","last_became_peered":"2026-02-22T21:45:01.133900+0000","last_unstale":"2026-02-22T22:09:41.658276+0000","last_undegraded":"2026-02-22T22:09:41.658276+0000","last_fullsized":"2026-02-22T22:09:41.658276+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_clean_scrub_stamp":"2026-02-22T21:44:59.253183+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:32:00.074720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705968,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":144904,"kb_used_data":1396,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":94226936,"statfs":{"total":96636764160,"available":96488382464,"internally_reserved":0,"allocated":1429504,"data_stored":713180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738673,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":159376,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":94212464,"statfs":{"total":96636764160,"available":96473563136,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161151430},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738675,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":163536,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":94208304,"statfs":{"total":96636764160,"available":96469303296,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:10:13.942 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:10:13.942 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:10:14.102 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T22:10:14.103 INFO:teuthology.orchestra.run.trial048.stderr:dumped all 2026-02-22T22:10:14.111 INFO:teuthology.orchestra.run.trial048.stdout:{"pg_ready":true,"pg_map":{"version":923,"stamp":"2026-02-22T22:10:12.744298+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":467816,"kb_used_data":5356,"kb_used_omap":4,"kb_used_meta":462395,"kb_avail":282647704,"statfs":{"total":289910292480,"available":289431248896,"internally_reserved":0,"allocated":5484544,"data_stored":3320314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":473492818},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000912"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660753+0000","last_change":"2026-02-22T21:45:06.268535+0000","last_active":"2026-02-22T22:09:41.660753+0000","last_peered":"2026-02-22T22:09:41.660753+0000","last_clean":"2026-02-22T22:09:41.660753+0000","last_became_active":"2026-02-22T21:45:04.266643+0000","last_became_peered":"2026-02-22T21:45:04.266643+0000","last_unstale":"2026-02-22T22:09:41.660753+0000","last_undegraded":"2026-02-22T22:09:41.660753+0000","last_fullsized":"2026-02-22T22:09:41.660753+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:35.211466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082153,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661288+0000","last_change":"2026-02-22T21:45:06.268006+0000","last_active":"2026-02-22T22:09:41.661288+0000","last_peered":"2026-02-22T22:09:41.661288+0000","last_clean":"2026-02-22T22:09:41.661288+0000","last_became_active":"2026-02-22T21:45:04.266954+0000","last_became_peered":"2026-02-22T21:45:04.266954+0000","last_unstale":"2026-02-22T22:09:41.661288+0000","last_undegraded":"2026-02-22T22:09:41.661288+0000","last_fullsized":"2026-02-22T22:09:41.661288+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:27:29.460040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030903099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.658947+0000","last_change":"2026-02-22T21:45:06.268671+0000","last_active":"2026-02-22T22:09:41.658947+0000","last_peered":"2026-02-22T22:09:41.658947+0000","last_clean":"2026-02-22T22:09:41.658947+0000","last_became_active":"2026-02-22T21:45:04.266721+0000","last_became_peered":"2026-02-22T21:45:04.266721+0000","last_unstale":"2026-02-22T22:09:41.658947+0000","last_undegraded":"2026-02-22T22:09:41.658947+0000","last_fullsized":"2026-02-22T22:09:41.658947+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:11:11.390548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094190300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660725+0000","last_change":"2026-02-22T21:45:06.268222+0000","last_active":"2026-02-22T22:09:41.660725+0000","last_peered":"2026-02-22T22:09:41.660725+0000","last_clean":"2026-02-22T22:09:41.660725+0000","last_became_active":"2026-02-22T21:45:04.266897+0000","last_became_peered":"2026-02-22T21:45:04.266897+0000","last_unstale":"2026-02-22T22:09:41.660725+0000","last_undegraded":"2026-02-22T22:09:41.660725+0000","last_fullsized":"2026-02-22T22:09:41.660725+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:10.132872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000509874,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:40.349127+0000","last_change":"2026-02-22T21:45:06.269931+0000","last_active":"2026-02-22T22:09:40.349127+0000","last_peered":"2026-02-22T22:09:40.349127+0000","last_clean":"2026-02-22T22:09:40.349127+0000","last_became_active":"2026-02-22T21:45:04.266353+0000","last_became_peered":"2026-02-22T21:45:04.266353+0000","last_unstale":"2026-02-22T22:09:40.349127+0000","last_undegraded":"2026-02-22T22:09:40.349127+0000","last_fullsized":"2026-02-22T22:09:40.349127+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:48:09.631675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080603100000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.539967+0000","last_change":"2026-02-22T21:45:06.795048+0000","last_active":"2026-02-22T22:09:41.539967+0000","last_peered":"2026-02-22T22:09:41.539967+0000","last_clean":"2026-02-22T22:09:41.539967+0000","last_became_active":"2026-02-22T21:45:04.266437+0000","last_became_peered":"2026-02-22T21:45:04.266437+0000","last_unstale":"2026-02-22T22:09:41.539967+0000","last_undegraded":"2026-02-22T22:09:41.539967+0000","last_fullsized":"2026-02-22T22:09:41.539967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:12:37.999670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021661099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.536537+0000","last_change":"2026-02-22T21:45:06.795071+0000","last_active":"2026-02-22T22:09:41.536537+0000","last_peered":"2026-02-22T22:09:41.536537+0000","last_clean":"2026-02-22T22:09:41.536537+0000","last_became_active":"2026-02-22T21:45:04.266381+0000","last_became_peered":"2026-02-22T21:45:04.266381+0000","last_unstale":"2026-02-22T22:09:41.536537+0000","last_undegraded":"2026-02-22T22:09:41.536537+0000","last_fullsized":"2026-02-22T22:09:41.536537+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:59.877077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022258099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661093+0000","last_change":"2026-02-22T21:45:06.268349+0000","last_active":"2026-02-22T22:09:41.661093+0000","last_peered":"2026-02-22T22:09:41.661093+0000","last_clean":"2026-02-22T22:09:41.661093+0000","last_became_active":"2026-02-22T21:45:04.266127+0000","last_became_peered":"2026-02-22T21:45:04.266127+0000","last_unstale":"2026-02-22T22:09:41.661093+0000","last_undegraded":"2026-02-22T22:09:41.661093+0000","last_fullsized":"2026-02-22T22:09:41.661093+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:39:02.311582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065919899999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":424,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.658276+0000","last_change":"2026-02-22T21:45:01.134350+0000","last_active":"2026-02-22T22:09:41.658276+0000","last_peered":"2026-02-22T22:09:41.658276+0000","last_clean":"2026-02-22T22:09:41.658276+0000","last_became_active":"2026-02-22T21:45:01.133900+0000","last_became_peered":"2026-02-22T21:45:01.133900+0000","last_unstale":"2026-02-22T22:09:41.658276+0000","last_undegraded":"2026-02-22T22:09:41.658276+0000","last_fullsized":"2026-02-22T22:09:41.658276+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_clean_scrub_stamp":"2026-02-22T21:44:59.253183+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:32:00.074720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705968,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":144904,"kb_used_data":1396,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":94226936,"statfs":{"total":96636764160,"available":96488382464,"internally_reserved":0,"allocated":1429504,"data_stored":713180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738673,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":159376,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":94212464,"statfs":{"total":96636764160,"available":96473563136,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161151430},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738675,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":163536,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":94208304,"statfs":{"total":96636764160,"available":96469303296,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:10:14.113 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:10:14.274 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T22:10:14.274 INFO:teuthology.orchestra.run.trial048.stdout:{"epoch":167,"fsid":"b0936238-8de0-4459-9a27-b7bc50f582a5","created":"2026-02-22T21:44:48.208030+0000","modified":"2026-02-22T22:09:40.345686+0000","last_up_change":"2026-02-22T21:44:58.250093+0000","last_in_change":"2026-02-22T21:44:48.785911+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T21:44:58.539685+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T21:45:02.395533+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"13807f88-98fa-4d56-b03f-7a5409e66ae9","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6802","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6803","nonce":2906375031}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6804","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6805","nonce":2906375031}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6808","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6809","nonce":2906375031}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6806","nonce":2906375031},{"type":"v1","addr":"10.20.193.48:6807","nonce":2906375031}]},"public_addr":"10.20.193.48:6803/2906375031","cluster_addr":"10.20.193.48:6805/2906375031","heartbeat_back_addr":"10.20.193.48:6809/2906375031","heartbeat_front_addr":"10.20.193.48:6807/2906375031","state":["exists","up"]},{"osd":1,"uuid":"b010d8db-2fbc-495e-bb8d-60a37b18f394","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6810","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6811","nonce":2771328344}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6812","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6813","nonce":2771328344}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6816","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6817","nonce":2771328344}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6814","nonce":2771328344},{"type":"v1","addr":"10.20.193.48:6815","nonce":2771328344}]},"public_addr":"10.20.193.48:6811/2771328344","cluster_addr":"10.20.193.48:6813/2771328344","heartbeat_back_addr":"10.20.193.48:6817/2771328344","heartbeat_front_addr":"10.20.193.48:6815/2771328344","state":["exists","up"]},{"osd":2,"uuid":"9121920a-57f3-4109-b3a3-b027b2d3ac5d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":102,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6818","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6819","nonce":3646627341}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6820","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6821","nonce":3646627341}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6824","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6825","nonce":3646627341}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.48:6822","nonce":3646627341},{"type":"v1","addr":"10.20.193.48:6823","nonce":3646627341}]},"public_addr":"10.20.193.48:6819/3646627341","cluster_addr":"10.20.193.48:6821/3646627341","heartbeat_back_addr":"10.20.193.48:6825/3646627341","heartbeat_front_addr":"10.20.193.48:6823/3646627341","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.239903+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.217985+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T21:44:56.265063+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"10.20.193.104:0/623978912":"2026-02-22T23:09:24.977894+0000","10.20.193.104:0/971624840":"2026-02-22T23:09:24.747763+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":3,"snaps":[{"begin":45,"length":2}]},{"pool":5,"snaps":[{"begin":3,"length":2}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:10:15.283 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-22T22:10:15.284 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:10:15.368 INFO:teuthology.orchestra.run.trial048.stdout:{ 2026-02-22T22:10:15.368 INFO:teuthology.orchestra.run.trial048.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:10:15.368 INFO:teuthology.orchestra.run.trial048.stdout:} 2026-02-22T22:10:15.374 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-22T22:10:15.534 INFO:teuthology.orchestra.run.trial048.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:10:15.543 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-22T22:10:15.543 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:10:15.625 INFO:teuthology.orchestra.run.trial048.stdout:{ 2026-02-22T22:10:15.625 INFO:teuthology.orchestra.run.trial048.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:10:15.625 INFO:teuthology.orchestra.run.trial048.stdout:} 2026-02-22T22:10:15.632 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-22T22:10:15.789 INFO:teuthology.orchestra.run.trial048.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:10:15.797 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-22T22:10:15.797 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:10:15.879 INFO:teuthology.orchestra.run.trial048.stdout:{ 2026-02-22T22:10:15.879 INFO:teuthology.orchestra.run.trial048.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:10:15.880 INFO:teuthology.orchestra.run.trial048.stdout:} 2026-02-22T22:10:15.885 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-22T22:10:16.044 INFO:teuthology.orchestra.run.trial048.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:10:16.053 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:10:16.210 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T22:10:16.210 INFO:teuthology.orchestra.run.trial048.stderr:dumped all 2026-02-22T22:10:16.218 INFO:teuthology.orchestra.run.trial048.stdout:{"pg_ready":true,"pg_map":{"version":924,"stamp":"2026-02-22T22:10:14.744499+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":467816,"kb_used_data":5356,"kb_used_omap":4,"kb_used_meta":462395,"kb_avail":282647704,"statfs":{"total":289910292480,"available":289431248896,"internally_reserved":0,"allocated":5484544,"data_stored":3320314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":473492818},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000990"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660753+0000","last_change":"2026-02-22T21:45:06.268535+0000","last_active":"2026-02-22T22:09:41.660753+0000","last_peered":"2026-02-22T22:09:41.660753+0000","last_clean":"2026-02-22T22:09:41.660753+0000","last_became_active":"2026-02-22T21:45:04.266643+0000","last_became_peered":"2026-02-22T21:45:04.266643+0000","last_unstale":"2026-02-22T22:09:41.660753+0000","last_undegraded":"2026-02-22T22:09:41.660753+0000","last_fullsized":"2026-02-22T22:09:41.660753+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:51:35.211466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082153,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":332,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661288+0000","last_change":"2026-02-22T21:45:06.268006+0000","last_active":"2026-02-22T22:09:41.661288+0000","last_peered":"2026-02-22T22:09:41.661288+0000","last_clean":"2026-02-22T22:09:41.661288+0000","last_became_active":"2026-02-22T21:45:04.266954+0000","last_became_peered":"2026-02-22T21:45:04.266954+0000","last_unstale":"2026-02-22T22:09:41.661288+0000","last_undegraded":"2026-02-22T22:09:41.661288+0000","last_fullsized":"2026-02-22T22:09:41.661288+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:27:29.460040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030903099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.659060+0000","last_change":"2026-02-22T21:45:06.268671+0000","last_active":"2026-02-22T22:09:41.659060+0000","last_peered":"2026-02-22T22:09:41.659060+0000","last_clean":"2026-02-22T22:09:41.659060+0000","last_became_active":"2026-02-22T21:45:04.266721+0000","last_became_peered":"2026-02-22T21:45:04.266721+0000","last_unstale":"2026-02-22T22:09:41.659060+0000","last_undegraded":"2026-02-22T22:09:41.659060+0000","last_fullsized":"2026-02-22T22:09:41.659060+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:11:11.390548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094190300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.660725+0000","last_change":"2026-02-22T21:45:06.268222+0000","last_active":"2026-02-22T22:09:41.660725+0000","last_peered":"2026-02-22T22:09:41.660725+0000","last_clean":"2026-02-22T22:09:41.660725+0000","last_became_active":"2026-02-22T21:45:04.266897+0000","last_became_peered":"2026-02-22T21:45:04.266897+0000","last_unstale":"2026-02-22T22:09:41.660725+0000","last_undegraded":"2026-02-22T22:09:41.660725+0000","last_fullsized":"2026-02-22T22:09:41.660725+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:10.132872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000509874,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:40.349127+0000","last_change":"2026-02-22T21:45:06.269931+0000","last_active":"2026-02-22T22:09:40.349127+0000","last_peered":"2026-02-22T22:09:40.349127+0000","last_clean":"2026-02-22T22:09:40.349127+0000","last_became_active":"2026-02-22T21:45:04.266353+0000","last_became_peered":"2026-02-22T21:45:04.266353+0000","last_unstale":"2026-02-22T22:09:40.349127+0000","last_undegraded":"2026-02-22T22:09:40.349127+0000","last_fullsized":"2026-02-22T22:09:40.349127+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:48:09.631675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00080603100000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.539967+0000","last_change":"2026-02-22T21:45:06.795048+0000","last_active":"2026-02-22T22:09:41.539967+0000","last_peered":"2026-02-22T22:09:41.539967+0000","last_clean":"2026-02-22T22:09:41.539967+0000","last_became_active":"2026-02-22T21:45:04.266437+0000","last_became_peered":"2026-02-22T21:45:04.266437+0000","last_unstale":"2026-02-22T22:09:41.539967+0000","last_undegraded":"2026-02-22T22:09:41.539967+0000","last_fullsized":"2026-02-22T22:09:41.539967+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:12:37.999670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021661099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":333,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.536537+0000","last_change":"2026-02-22T21:45:06.795071+0000","last_active":"2026-02-22T22:09:41.536537+0000","last_peered":"2026-02-22T22:09:41.536537+0000","last_clean":"2026-02-22T22:09:41.536537+0000","last_became_active":"2026-02-22T21:45:04.266381+0000","last_became_peered":"2026-02-22T21:45:04.266381+0000","last_unstale":"2026-02-22T22:09:41.536537+0000","last_undegraded":"2026-02-22T22:09:41.536537+0000","last_fullsized":"2026-02-22T22:09:41.536537+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:59.877077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022258099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":334,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.661093+0000","last_change":"2026-02-22T21:45:06.268349+0000","last_active":"2026-02-22T22:09:41.661093+0000","last_peered":"2026-02-22T22:09:41.661093+0000","last_clean":"2026-02-22T22:09:41.661093+0000","last_became_active":"2026-02-22T21:45:04.266127+0000","last_became_peered":"2026-02-22T21:45:04.266127+0000","last_unstale":"2026-02-22T22:09:41.661093+0000","last_undegraded":"2026-02-22T22:09:41.661093+0000","last_fullsized":"2026-02-22T22:09:41.661093+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:45:03.258112+0000","last_clean_scrub_stamp":"2026-02-22T21:45:03.258112+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:39:02.311582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065919899999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":424,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:09:41.658276+0000","last_change":"2026-02-22T21:45:01.134350+0000","last_active":"2026-02-22T22:09:41.658276+0000","last_peered":"2026-02-22T22:09:41.658276+0000","last_clean":"2026-02-22T22:09:41.658276+0000","last_became_active":"2026-02-22T21:45:01.133900+0000","last_became_peered":"2026-02-22T21:45:01.133900+0000","last_unstale":"2026-02-22T22:09:41.658276+0000","last_undegraded":"2026-02-22T22:09:41.658276+0000","last_fullsized":"2026-02-22T22:09:41.658276+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T21:44:59.253183+0000","last_clean_scrub_stamp":"2026-02-22T21:44:59.253183+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:32:00.074720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705969,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":144904,"kb_used_data":1396,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":94226936,"statfs":{"total":96636764160,"available":96488382464,"internally_reserved":0,"allocated":1429504,"data_stored":713180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738674,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":159376,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":94212464,"statfs":{"total":96636764160,"available":96473563136,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161151430},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738676,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":163536,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":94208304,"statfs":{"total":96636764160,"available":96469303296,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T21:45:03.258112+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=45, tm_sec=3, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T21:44:59.253183+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=21, tm_min=44, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=10, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:10:16.219 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:10:36.219 DEBUG:teuthology.orchestra.run.trial048:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:10:36.381 INFO:teuthology.orchestra.run.trial048.stdout: 2026-02-22T22:10:36.381 INFO:teuthology.orchestra.run.trial048.stderr:dumped all 2026-02-22T22:10:36.389 INFO:teuthology.orchestra.run.trial048.stdout:{"pg_ready":true,"pg_map":{"version":934,"stamp":"2026-02-22T22:10:34.746314+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":467804,"kb_used_data":5356,"kb_used_omap":4,"kb_used_meta":462395,"kb_avail":282647716,"statfs":{"total":289910292480,"available":289431261184,"internally_reserved":0,"allocated":5484544,"data_stored":3320314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":473492818},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000938"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":341,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:18.599145+0000","last_change":"2026-02-22T22:10:18.599157+0000","last_active":"2026-02-22T22:10:18.599145+0000","last_peered":"2026-02-22T22:10:18.599145+0000","last_clean":"2026-02-22T22:10:18.599145+0000","last_became_active":"2026-02-22T21:45:04.266643+0000","last_became_peered":"2026-02-22T21:45:04.266643+0000","last_unstale":"2026-02-22T22:10:18.599145+0000","last_undegraded":"2026-02-22T22:10:18.599145+0000","last_fullsized":"2026-02-22T22:10:18.599145+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:10:18.599096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:10:18.599096+0000","last_clean_scrub_stamp":"2026-02-22T22:10:18.599096+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:55:36.691796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082153,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":341,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:21.738053+0000","last_change":"2026-02-22T22:10:21.738065+0000","last_active":"2026-02-22T22:10:21.738053+0000","last_peered":"2026-02-22T22:10:21.738053+0000","last_clean":"2026-02-22T22:10:21.738053+0000","last_became_active":"2026-02-22T21:45:04.266954+0000","last_became_peered":"2026-02-22T21:45:04.266954+0000","last_unstale":"2026-02-22T22:10:21.738053+0000","last_undegraded":"2026-02-22T22:10:21.738053+0000","last_fullsized":"2026-02-22T22:10:21.738053+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:10:21.738007+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:10:21.738007+0000","last_clean_scrub_stamp":"2026-02-22T22:10:21.738007+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:20:27.234714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030903099999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":342,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:19.639991+0000","last_change":"2026-02-22T22:10:19.640007+0000","last_active":"2026-02-22T22:10:19.639991+0000","last_peered":"2026-02-22T22:10:19.639991+0000","last_clean":"2026-02-22T22:10:19.639991+0000","last_became_active":"2026-02-22T21:45:04.266721+0000","last_became_peered":"2026-02-22T21:45:04.266721+0000","last_unstale":"2026-02-22T22:10:19.639991+0000","last_undegraded":"2026-02-22T22:10:19.639991+0000","last_fullsized":"2026-02-22T22:10:19.639991+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:10:19.639943+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:10:19.639943+0000","last_clean_scrub_stamp":"2026-02-22T22:10:19.639943+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:57:11.060499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094190300000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":342,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:20.689391+0000","last_change":"2026-02-22T22:10:20.689404+0000","last_active":"2026-02-22T22:10:20.689391+0000","last_peered":"2026-02-22T22:10:20.689391+0000","last_clean":"2026-02-22T22:10:20.689391+0000","last_became_active":"2026-02-22T21:45:04.266897+0000","last_became_peered":"2026-02-22T21:45:04.266897+0000","last_unstale":"2026-02-22T22:10:20.689391+0000","last_undegraded":"2026-02-22T22:10:20.689391+0000","last_fullsized":"2026-02-22T22:10:20.689391+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:10:20.689338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:10:20.689338+0000","last_clean_scrub_stamp":"2026-02-22T22:10:20.689338+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:01:47.723318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000509874,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"167'3","reported_seq":344,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:15.554543+0000","last_change":"2026-02-22T22:10:15.554561+0000","last_active":"2026-02-22T22:10:15.554543+0000","last_peered":"2026-02-22T22:10:15.554543+0000","last_clean":"2026-02-22T22:10:15.554543+0000","last_became_active":"2026-02-22T21:45:04.266353+0000","last_became_peered":"2026-02-22T21:45:04.266353+0000","last_unstale":"2026-02-22T22:10:15.554543+0000","last_undegraded":"2026-02-22T22:10:15.554543+0000","last_fullsized":"2026-02-22T22:10:15.554543+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"167'3","last_scrub_stamp":"2026-02-22T22:10:15.554489+0000","last_deep_scrub":"167'3","last_deep_scrub_stamp":"2026-02-22T22:10:15.554489+0000","last_clean_scrub_stamp":"2026-02-22T22:10:15.554489+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:10:32.044564+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00080603100000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":342,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:17.524288+0000","last_change":"2026-02-22T22:10:17.524299+0000","last_active":"2026-02-22T22:10:17.524288+0000","last_peered":"2026-02-22T22:10:17.524288+0000","last_clean":"2026-02-22T22:10:17.524288+0000","last_became_active":"2026-02-22T21:45:04.266437+0000","last_became_peered":"2026-02-22T21:45:04.266437+0000","last_unstale":"2026-02-22T22:10:17.524288+0000","last_undegraded":"2026-02-22T22:10:17.524288+0000","last_fullsized":"2026-02-22T22:10:17.524288+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:10:17.524253+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:10:17.524253+0000","last_clean_scrub_stamp":"2026-02-22T22:10:17.524253+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:00:48.398722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021661099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":342,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:16.489103+0000","last_change":"2026-02-22T22:10:16.489112+0000","last_active":"2026-02-22T22:10:16.489103+0000","last_peered":"2026-02-22T22:10:16.489103+0000","last_clean":"2026-02-22T22:10:16.489103+0000","last_became_active":"2026-02-22T21:45:04.266381+0000","last_became_peered":"2026-02-22T21:45:04.266381+0000","last_unstale":"2026-02-22T22:10:16.489103+0000","last_undegraded":"2026-02-22T22:10:16.489103+0000","last_fullsized":"2026-02-22T22:10:16.489103+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:10:16.489061+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:10:16.489061+0000","last_clean_scrub_stamp":"2026-02-22T22:10:16.489061+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:30:26.286877+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00022258099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"167'2","reported_seq":344,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:17.566095+0000","last_change":"2026-02-22T22:10:17.566113+0000","last_active":"2026-02-22T22:10:17.566095+0000","last_peered":"2026-02-22T22:10:17.566095+0000","last_clean":"2026-02-22T22:10:17.566095+0000","last_became_active":"2026-02-22T21:45:04.266127+0000","last_became_peered":"2026-02-22T21:45:04.266127+0000","last_unstale":"2026-02-22T22:10:17.566095+0000","last_undegraded":"2026-02-22T22:10:17.566095+0000","last_fullsized":"2026-02-22T22:10:17.566095+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"167'2","last_scrub_stamp":"2026-02-22T22:10:17.566055+0000","last_deep_scrub":"167'2","last_deep_scrub_stamp":"2026-02-22T22:10:17.566055+0000","last_clean_scrub_stamp":"2026-02-22T22:10:17.566055+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:41:22.736394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00065919899999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"167'74","reported_seq":435,"reported_epoch":167,"state":"active+clean","last_fresh":"2026-02-22T22:10:16.557928+0000","last_change":"2026-02-22T22:10:16.557942+0000","last_active":"2026-02-22T22:10:16.557928+0000","last_peered":"2026-02-22T22:10:16.557928+0000","last_clean":"2026-02-22T22:10:16.557928+0000","last_became_active":"2026-02-22T21:45:01.133900+0000","last_became_peered":"2026-02-22T21:45:01.133900+0000","last_unstale":"2026-02-22T22:10:16.557928+0000","last_undegraded":"2026-02-22T22:10:16.557928+0000","last_fullsized":"2026-02-22T22:10:16.557928+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"167'74","last_scrub_stamp":"2026-02-22T22:10:16.557903+0000","last_deep_scrub":"167'74","last_deep_scrub_stamp":"2026-02-22T22:10:16.557903+0000","last_clean_scrub_stamp":"2026-02-22T22:10:16.557903+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:43:11.782890+0000","scrub_duration":3,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705973,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":144900,"kb_used_data":1396,"kb_used_omap":1,"kb_used_meta":143486,"kb_avail":94226940,"statfs":{"total":96636764160,"available":96488386560,"internally_reserved":0,"allocated":1429504,"data_stored":713180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":146930118},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738678,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":159372,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":157374,"kb_avail":94212468,"statfs":{"total":96636764160,"available":96473567232,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":161151430},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738680,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":163532,"kb_used_data":1980,"kb_used_omap":1,"kb_used_meta":161534,"kb_avail":94208308,"statfs":{"total":96636764160,"available":96469307392,"internally_reserved":0,"allocated":2027520,"data_stored":1303567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":165411270},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:10:36.390 DEBUG:teuthology.orchestra.run.trial048:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-22T22:10:36.555 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:10:36.555 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:10:36.555 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-22T22:10:36.555 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:36.607 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-22T22:10:36.608 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-22T22:10:36.608 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:36.651 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-22T22:10:36.651 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-22T22:10:36.651 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:36.688 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-22T22:10:36.688 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:10:36.688 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-22T22:10:36.688 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:36.760 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-22T22:10:36.760 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:10:36.760 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-22T22:10:36.760 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:10:36.788 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 is failed for ~0s 2026-02-22T22:10:36.812 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-22T22:10:36.812 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:10:36.812 DEBUG:teuthology.orchestra.run.trial048:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:10:36.878 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial048.front.sepia.ceph.com 2026-02-22T22:10:36.878 DEBUG:teuthology.orchestra.run.trial048:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-22T22:10:36.953 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial048.front.sepia.ceph.com 2026-02-22T22:10:36.953 DEBUG:teuthology.orchestra.run.trial048:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-22T22:10:37.027 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial048.front.sepia.ceph.com 2026-02-22T22:10:37.028 DEBUG:teuthology.orchestra.run.trial048:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-22T22:10:37.107 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:10:37.107 DEBUG:teuthology.misc:Transferring archived files from trial048:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/data/cluster1.mon.a.tgz 2026-02-22T22:10:37.108 DEBUG:teuthology.orchestra.run.trial048:> mktemp 2026-02-22T22:10:37.162 INFO:teuthology.orchestra.run.trial048.stdout:/tmp/tmp.NBJfVf2mst 2026-02-22T22:10:37.163 DEBUG:teuthology.orchestra.run.trial048:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.NBJfVf2mst 2026-02-22T22:10:37.275 DEBUG:teuthology.orchestra.run.trial048:> sudo chmod 0666 /tmp/tmp.NBJfVf2mst 2026-02-22T22:10:37.355 DEBUG:teuthology.orchestra.remote:trial048:/tmp/tmp.NBJfVf2mst is 413KB 2026-02-22T22:10:37.416 DEBUG:teuthology.orchestra.run.trial048:> rm -fr /tmp/tmp.NBJfVf2mst 2026-02-22T22:10:37.433 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:10:37.433 DEBUG:teuthology.orchestra.run.trial048:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:10:37.476 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:10:37.556 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:10:37.573 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:10:37.573 INFO:tasks.ceph:Archiving crash dumps... 2026-02-22T22:10:37.577 DEBUG:teuthology.misc:Transferring archived files from trial048:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/remote/trial048/crash 2026-02-22T22:10:37.578 DEBUG:teuthology.orchestra.run.trial048:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:10:37.620 DEBUG:teuthology.misc:Transferring archived files from trial104:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/remote/trial104/crash 2026-02-22T22:10:37.622 DEBUG:teuthology.orchestra.run.trial104:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:10:37.646 INFO:tasks.ceph:Compressing logs... 2026-02-22T22:10:37.646 DEBUG:teuthology.orchestra.run.trial048:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:10:37.665 DEBUG:teuthology.orchestra.run.trial104:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:10:37.685 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46027.log 2026-02-22T22:10:37.686 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47270.log 2026-02-22T22:10:37.686 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46027.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46027.log.gz 2026-02-22T22:10:37.686 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.37305.log 2026-02-22T22:10:37.686 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47270.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45586.log 2026-02-22T22:10:37.686 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47270.log.gz 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/ceph.tmp-client.admin.37305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47368.log 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.37305.log.gz 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45586.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45426.log 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45586.log.gz 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47368.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47368.log.gz 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45426.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46235.log 2026-02-22T22:10:37.687 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45426.log.gz 2026-02-22T22:10:37.688 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46174.log 2026-02-22T22:10:37.688 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46235.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster1-client.admin.46235.log.gz --verbose 2026-02-22T22:10:37.688 INFO:teuthology.orchestra.run.trial048.stderr: -- /var/log/ceph/cluster1-client.admin.47662.log 2026-02-22T22:10:37.688 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-22T22:10:37.688 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46174.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46174.log.gz 2026-02-22T22:10:37.689 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42207.log 2026-02-22T22:10:37.689 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47662.log.gz 2026-02-22T22:10:37.689 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-22T22:10:37.689 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.42207.log: gzip 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42207.log.gz -5 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.42392.log 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45907.log 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.42392.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42392.log.gz 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47613.log 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45907.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47709.log 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45907.log.gz 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47613.log.gz 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47466.log 2026-02-22T22:10:37.690 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47709.log.gz 2026-02-22T22:10:37.691 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45335.log 2026-02-22T22:10:37.691 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47466.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47466.log.gz 2026-02-22T22:10:37.691 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47123.log 2026-02-22T22:10:37.691 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45335.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46370.log 2026-02-22T22:10:37.691 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45335.log.gz 2026-02-22T22:10:37.691 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47123.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47417.log 2026-02-22T22:10:37.692 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47123.log.gz 2026-02-22T22:10:37.692 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46370.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46370.log.gz 2026-02-22T22:10:37.692 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-22T22:10:37.692 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47319.log 2026-02-22T22:10:37.692 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47417.log.gz 2026-02-22T22:10:37.692 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47515.log 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47319.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45906.log 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47319.log.gz 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47515.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47515.log.gz 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45521.log 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45906.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45785.log 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45906.log.gz 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45521.log: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.45521.log.gz 2026-02-22T22:10:37.693 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47221.log 2026-02-22T22:10:37.694 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45785.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45787.log 2026-02-22T22:10:37.694 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45785.log.gz 2026-02-22T22:10:37.694 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47221.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46090.log 2026-02-22T22:10:37.694 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47221.log.gz 2026-02-22T22:10:37.694 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45787.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45787.log.gz 2026-02-22T22:10:37.694 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45786.log 2026-02-22T22:10:37.694 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46090.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46419.log 2026-02-22T22:10:37.695 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46090.log.gz 2026-02-22T22:10:37.695 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45786.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45786.log.gz 2026-02-22T22:10:37.695 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-22T22:10:37.695 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46419.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46419.log.gz 2026-02-22T22:10:37.695 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42681.log 2026-02-22T22:10:37.695 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46231.log 2026-02-22T22:10:37.696 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.42681.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42486.log 2026-02-22T22:10:37.696 INFO:teuthology.orchestra.run.trial048.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42681.log.gz 2026-02-22T22:10:37.696 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46231.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46231.log.gz 2026-02-22T22:10:37.696 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42439.log 2026-02-22T22:10:37.696 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.42486.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42486.log.gz 2026-02-22T22:10:37.696 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-22T22:10:37.696 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.42439.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42439.log.gz 2026-02-22T22:10:37.697 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47564.log 2026-02-22T22:10:37.697 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45684.log 2026-02-22T22:10:37.697 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47564.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47564.log.gz 2026-02-22T22:10:37.697 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47172.log 2026-02-22T22:10:37.697 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45684.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45684.log.gz 2026-02-22T22:10:37.697 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45635.log 2026-02-22T22:10:37.697 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.47172.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47172.log.gz 2026-02-22T22:10:37.698 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45496.log 2026-02-22T22:10:37.698 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45635.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45635.log.gz 2026-02-22T22:10:37.698 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42613.log 2026-02-22T22:10:37.698 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45496.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45496.log.gz 2026-02-22T22:10:37.698 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46089.log 2026-02-22T22:10:37.698 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.42613.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42613.log.gz 2026-02-22T22:10:37.698 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-22T22:10:37.699 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46089.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46089.log.gz 2026-02-22T22:10:37.699 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45952.log 2026-02-22T22:10:37.699 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46321.log 2026-02-22T22:10:37.699 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.45952.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45952.log.gz 2026-02-22T22:10:37.699 INFO:teuthology.orchestra.run.trial048.stderr:/var/log/ceph/cluster1-client.admin.46321.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46321.log.gz 2026-02-22T22:10:37.700 INFO:teuthology.orchestra.run.trial048.stderr: 87.9% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-22T22:10:37.702 INFO:teuthology.orchestra.run.trial048.stderr: 93.7% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-22T22:10:37.711 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56626.log 2026-02-22T22:10:37.711 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71329.log 2026-02-22T22:10:37.711 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61116.log 2026-02-22T22:10:37.711 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.71329.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.56626.log.gz 2026-02-22T22:10:37.712 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79975.log 2026-02-22T22:10:37.712 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.61116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51300.log 2026-02-22T22:10:37.712 INFO:teuthology.orchestra.run.trial104.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.71329.log.gz 2026-02-22T22:10:37.712 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.79975.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94047.log 2026-02-22T22:10:37.712 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51300.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.61116.log.gz 2026-02-22T22:10:37.712 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.44128.log 2026-02-22T22:10:37.712 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51300.log.gz 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94047.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61626.log 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.44128.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.44128.log.gz 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.94047.log.gz 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71480.log 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.79975.log.gz 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61626.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47097.log 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.61626.log.gz 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79685.log 2026-02-22T22:10:37.713 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.71480.log.gz 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47097.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47097.log.gz 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92888.log 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47832.log 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79685.log.gz 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92888.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.95658.log 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92888.log.gz 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47832.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47832.log.gz 2026-02-22T22:10:37.714 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55838.log 2026-02-22T22:10:37.715 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.95658.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67762.log 2026-02-22T22:10:37.715 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.55838.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69306.log 2026-02-22T22:10:37.715 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.55838.log.gz 2026-02-22T22:10:37.715 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76342.log 2026-02-22T22:10:37.715 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.67762.log.gz 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.69306.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49068.log 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.69306.log.gz 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61660.log 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76342.log: /var/log/ceph/cluster1-client.mirror.49068.log: 63.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49068.log.gz 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89928.log 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.76342.log.gz 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.95658.log.gz 2026-02-22T22:10:37.716 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70407.log 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.89928.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.61660.log.gz 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74064.log 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.89928.log.gz 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr:gzip/var/log/ceph/cluster1-client.mirror.70407.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.69407.log 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74064.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67553.log 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.74064.log.gz 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69407.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.70407.log.gz 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77905.log 2026-02-22T22:10:37.717 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.69407.log.gz 2026-02-22T22:10:37.718 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67553.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73711.log 2026-02-22T22:10:37.718 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.67553.log.gz 2026-02-22T22:10:37.718 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.77905.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68647.log 2026-02-22T22:10:37.718 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73711.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87367.log 2026-02-22T22:10:37.718 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68647.log: 88.2%gzip -- replaced with /var/log/ceph/cluster1-client.admin.73711.log.gz 2026-02-22T22:10:37.718 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.93817.log 2026-02-22T22:10:37.718 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.68647.log.gz 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87367.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48891.log 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.77905.log.gz 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94555.log 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.93817.log.gz 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48891.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62219.log 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48891.log.gz 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.94555.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94555.log.gz 2026-02-22T22:10:37.719 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52697.log 2026-02-22T22:10:37.720 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62219.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63442.log 2026-02-22T22:10:37.720 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52697.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.62219.log.gz 2026-02-22T22:10:37.720 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63167.log 2026-02-22T22:10:37.720 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63442.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47467.log 2026-02-22T22:10:37.720 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.52697.log.gz 2026-02-22T22:10:37.720 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.63442.log.gz 2026-02-22T22:10:37.720 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76594.log 2026-02-22T22:10:37.721 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47467.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.63167.log.gz 2026-02-22T22:10:37.721 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47467.log.gz 2026-02-22T22:10:37.721 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74954.log 2026-02-22T22:10:37.721 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.76594.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76594.log.gz 2026-02-22T22:10:37.721 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55521.log 2026-02-22T22:10:37.721 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74954.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55685.log 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.55521.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.74954.log.gz 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67459.log 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.55521.log.gz 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.55685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89069.log 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.55685.log.gz 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.67459.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49779.log 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.67459.log.gz 2026-02-22T22:10:37.722 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89069.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60574.log 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89069.log.gz 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89205.log 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60574.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49779.log.gz 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55956.log 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.60574.log.gz 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89205.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92501.log 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89205.log.gz 2026-02-22T22:10:37.723 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.55956.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49034.log 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.55956.log.gz/var/log/ceph/cluster2-client.mirror.92501.log: 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92669.log 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.49034.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75529.log 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.92501.log.gz 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr: 63.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49034.log.gz 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86732.log 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92669.log.gz 2026-02-22T22:10:37.724 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75529.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95029.log 2026-02-22T22:10:37.725 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.86732.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.75529.log.gz 2026-02-22T22:10:37.725 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96177.log 2026-02-22T22:10:37.725 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.86732.log.gz 2026-02-22T22:10:37.725 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.95029.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.38548.log 2026-02-22T22:10:37.725 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.95029.log.gz 2026-02-22T22:10:37.725 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.96177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64830.log 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr: 88.7% -- replaced with /var/log/ceph/cluster1-client.admin.96177.log.gz 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/ceph.tmp-client.admin.38548.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60514.log 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.38548.log.gz 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64830.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48322.log 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.60514.log: gzip 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.64830.log.gz -5 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.54336.log 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.60514.log.gz 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48322.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster2-client.admin.65948.log 2026-02-22T22:10:37.726 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.48322.log.gz 2026-02-22T22:10:37.727 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71662.log 2026-02-22T22:10:37.727 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54336.log.gz 2026-02-22T22:10:37.727 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74772.log 2026-02-22T22:10:37.727 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71662.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.65948.log.gz 2026-02-22T22:10:37.727 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69796.log 2026-02-22T22:10:37.727 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.71662.log.gz 2026-02-22T22:10:37.727 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74772.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57685.log 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.74772.log.gz 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.69796.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79472.log 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69796.log.gz 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86336.log 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57685.log.gz 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79472.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82613.log 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79472.log.gz 2026-02-22T22:10:37.728 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86336.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69678.log 2026-02-22T22:10:37.729 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86336.log.gz 2026-02-22T22:10:37.729 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93846.log 2026-02-22T22:10:37.729 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.69678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75707.log 2026-02-22T22:10:37.729 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.93846.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88504.log 2026-02-22T22:10:37.729 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.69678.log.gz 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.93846.log.gz 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75707.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68710.log 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82613.log.gz 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88504.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.75707.log.gz 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60481.log 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88504.log.gz 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68710.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82103.log 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.68710.log.gz 2026-02-22T22:10:37.730 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.60481.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55024.log 2026-02-22T22:10:37.731 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82103.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64298.log 88.6% 2026-02-22T22:10:37.731 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.60481.log.gz 2026-02-22T22:10:37.731 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.55024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53964.log 2026-02-22T22:10:37.731 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64298.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.55024.log.gz 2026-02-22T22:10:37.731 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85519.log 2026-02-22T22:10:37.731 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.82103.log.gz 2026-02-22T22:10:37.731 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.64298.log.gz 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.53964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78256.log 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53964.log.gz 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77311.log 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85519.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.87367.log.gz 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78256.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74987.log 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77311.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80826.log 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78256.log.gz 2026-02-22T22:10:37.732 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74987.log: 88.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77311.log.gz 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47199.log 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.74987.log.gz 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80826.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85519.log.gz 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89563.log 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47199.log: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80826.log.gz 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47199.log.gz 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-22T22:10:37.733 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89563.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88266.log 2026-02-22T22:10:37.734 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2.log: gzip 87.4% -5 -- replaced with /var/log/ceph/cluster1-client.mirror.89563.log.gz --verbose -- 2026-02-22T22:10:37.734 INFO:teuthology.orchestra.run.trial104.stderr: /var/log/ceph/cluster2-client.mirror.0.94100.log 2026-02-22T22:10:37.734 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73229.log 2026-02-22T22:10:37.734 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.0.94100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55810.log 2026-02-22T22:10:37.734 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.94100.log.gz 2026-02-22T22:10:37.734 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73229.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96479.log 2026-02-22T22:10:37.735 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.55810.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.73229.log.gz 2026-02-22T22:10:37.735 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84920.log 2026-02-22T22:10:37.735 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.55810.log.gz 2026-02-22T22:10:37.735 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.96479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82696.log 2026-02-22T22:10:37.735 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64861.log 2026-02-22T22:10:37.735 INFO:teuthology.orchestra.run.trial104.stderr: 87.7% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-22T22:10:37.736 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.82696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80179.log 2026-02-22T22:10:37.736 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.82696.log.gz 2026-02-22T22:10:37.736 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84920.log.gz 2026-02-22T22:10:37.736 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86763.log 2026-02-22T22:10:37.736 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.64861.log.gz 2026-02-22T22:10:37.736 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80179.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65797.log 2026-02-22T22:10:37.736 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75997.log 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.86763.log: /var/log/ceph/cluster1-client.mirror.65797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66163.log 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75997.log: 90.4%gzip -- replaced with /var/log/ceph/cluster1-client.admin.86763.log.gz -5 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.91248.log 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.65797.log.gz 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.75997.log.gz 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.66163.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71755.log 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.66163.log.gz 2026-02-22T22:10:37.737 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91248.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61172.log 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91248.log.gz 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5/var/log/ceph/cluster1-client.admin.71755.log: --verbose -- /var/log/ceph/cluster2-client.mirror.85074.log 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88266.log.gz 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61172.log: 87.0%gzip -- replaced with /var/log/ceph/cluster1-client.admin.71755.log.gz -5 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.76954.log 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr: 95.6% -- replaced with /var/log/ceph/cluster2-client.mirror.96479.log.gz 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.61172.log.gz 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85074.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68560.log 2026-02-22T22:10:37.738 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76954.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.94704.log 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85074.log.gz 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.68560.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.76954.log.gz 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66251.log 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.94704.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65494.log 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.94704.log.gz 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68560.log.gz 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.66251.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65857.log 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65494.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66373.log 2026-02-22T22:10:37.739 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66251.log.gz 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.65494.log.gz 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.80179.log.gz 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65857.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89459.log 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.65857.log.gz 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94496.log 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.66373.log: /var/log/ceph/cluster1-client.mirror.89459.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.66373.log.gz 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64695.log 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.89459.log.gz 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.94496.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94496.log.gz 2026-02-22T22:10:37.740 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73020.log 2026-02-22T22:10:37.741 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.64695.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57277.log 2026-02-22T22:10:37.741 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64695.log.gz 2026-02-22T22:10:37.741 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.73020.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85184.log 2026-02-22T22:10:37.741 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73020.log.gz 2026-02-22T22:10:37.741 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.57277.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52446.log 2026-02-22T22:10:37.741 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.57277.log.gz 2026-02-22T22:10:37.742 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49929.log 2026-02-22T22:10:37.742 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85184.log: /var/log/ceph/cluster1-client.admin.52446.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79302.log 2026-02-22T22:10:37.742 INFO:teuthology.orchestra.run.trial104.stderr: 90.1%/var/log/ceph/cluster2-client.mirror.49929.log: -- replaced with /var/log/ceph/cluster1-client.admin.52446.log.gz 2026-02-22T22:10:37.742 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85184.log.gz 2026-02-22T22:10:37.742 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91964.log 2026-02-22T22:10:37.742 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49929.log.gz 2026-02-22T22:10:37.742 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82520.log 2026-02-22T22:10:37.743 INFO:teuthology.orchestra.run.trial104.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79302.log.gz 2026-02-22T22:10:37.743 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.90243.log 2026-02-22T22:10:37.743 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82520.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89859.log 2026-02-22T22:10:37.743 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.91964.log.gz 2026-02-22T22:10:37.743 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82520.log.gz 2026-02-22T22:10:37.743 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.90243.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77031.log 2026-02-22T22:10:37.743 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61720.log 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89859.log: /var/log/ceph/cluster1-client.mirror.77031.log: gzip -5 87.4% --verbose -- -- replaced with /var/log/ceph/cluster1-client.mirror.89859.log.gz /var/log/ceph/cluster1-client.admin.56956.log 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.77031.log.gz 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61720.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86940.log 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56956.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.61720.log.gz 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49222.log 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.56956.log.gz 2026-02-22T22:10:37.744 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.86940.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76275.log 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91183.log 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49222.log: 56.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49222.log.gz 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.86940.log.gz 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.76275.log: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.2.90243.log.gz 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70527.log 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76275.log.gz 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81287.log 2026-02-22T22:10:37.745 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91183.log.gz 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73935.log 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70527.log: /var/log/ceph/cluster2-client.mirror.81287.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90669.log 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.70527.log.gz 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.73935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47662.log 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81287.log.gz 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73935.log.gz 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58423.log 2026-02-22T22:10:37.746 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.90669.log.gz 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70586.log 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47662.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47662.log.gz 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58423.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79226.log 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.58423.log.gz 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.70586.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.87471.log 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.70586.log.gz 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79226.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80442.log 2026-02-22T22:10:37.747 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.87471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.83588.log 2026-02-22T22:10:37.748 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.92021.log 2026-02-22T22:10:37.748 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.83588.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80442.log.gz 2026-02-22T22:10:37.748 INFO:teuthology.orchestra.run.trial104.stderr:gzip 0.0% -5 --verbose -- replaced with /var/log/ceph/cluster2-client.admin.83588.log.gz -- 2026-02-22T22:10:37.748 INFO:teuthology.orchestra.run.trial104.stderr: /var/log/ceph/cluster2-client.mirror.54060.log 2026-02-22T22:10:37.748 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.92021.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53349.log 2026-02-22T22:10:37.748 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54060.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.92021.log.gz 2026-02-22T22:10:37.749 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74302.log 2026-02-22T22:10:37.749 INFO:teuthology.orchestra.run.trial104.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54060.log.gz 2026-02-22T22:10:37.749 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53349.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48949.log 2026-02-22T22:10:37.749 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.87471.log.gz 2026-02-22T22:10:37.749 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.74302.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49290.log 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48949.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74302.log.gz 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr: 70.5% -- replaced with /var/log/ceph/cluster1-client.admin.48949.log.gz 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64391.log 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49290.log: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53349.log.gz 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82769.log 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr: 64.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49290.log.gz 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64391.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71724.log 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82769.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.64391.log.gz 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81473.log 2026-02-22T22:10:37.750 INFO:teuthology.orchestra.run.trial104.stderr: 56.7% -- replaced with /var/log/ceph/cluster1-client.mirror.82769.log.gz 2026-02-22T22:10:37.751 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.71724.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56766.log 2026-02-22T22:10:37.751 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.79226.log.gz 2026-02-22T22:10:37.751 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81473.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.71724.log.gz 2026-02-22T22:10:37.751 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81829.log 2026-02-22T22:10:37.751 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75837.log 2026-02-22T22:10:37.751 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56766.log: 87.5%/var/log/ceph/cluster1-client.mirror.81829.log: -- replaced with /var/log/ceph/cluster1-client.mirror.56766.log.gz 2026-02-22T22:10:37.751 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72506.log 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81829.log.gz 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.75837.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74800.log 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81473.log.gz 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75837.log.gz 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.72506.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63411.log 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.74800.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72506.log.gz 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49085.log 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63411.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.74800.log.gz 2026-02-22T22:10:37.752 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62188.log 2026-02-22T22:10:37.753 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.63411.log.gz 2026-02-22T22:10:37.753 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.49085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79395.log 2026-02-22T22:10:37.753 INFO:teuthology.orchestra.run.trial104.stderr: 54.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49085.log.gz 2026-02-22T22:10:37.753 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82786.log 2026-02-22T22:10:37.753 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79395.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.62188.log.gz 2026-02-22T22:10:37.753 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79269.log 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79395.log.gz 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82786.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72221.log 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr: 54.5% -- replaced with /var/log/ceph/cluster2-client.mirror.82786.log.gz 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79269.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94679.log 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.72221.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86095.log 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.94679.log: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.72221.log.gz 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65888.log 2026-02-22T22:10:37.754 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.94679.log.gz 2026-02-22T22:10:37.755 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49964.log 2026-02-22T22:10:37.755 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78729.log 2026-02-22T22:10:37.755 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65888.log: /var/log/ceph/cluster2-client.mirror.49964.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.86095.log.gz 2026-02-22T22:10:37.755 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.65888.log.gz 2026-02-22T22:10:37.755 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61515.log 2026-02-22T22:10:37.755 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49964.log.gz 2026-02-22T22:10:37.756 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49102.log 2026-02-22T22:10:37.756 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78729.log.gz 2026-02-22T22:10:37.756 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61515.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70558.log 2026-02-22T22:10:37.757 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.61515.log.gz 2026-02-22T22:10:37.757 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94580.log 2026-02-22T22:10:37.757 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.49102.log: 54.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49102.log.gz 2026-02-22T22:10:37.757 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61850.log 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70558.log: /var/log/ceph/cluster1-client.mirror.94580.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88958.log 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94580.log.gz 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.70558.log.gz 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.61850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76310.log 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.61850.log.gz 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.88958.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.94215.log 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88958.log.gz 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.76310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54899.log 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.3.94215.log: 87.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.76310.log.gz -5 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.51007.log 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr: 20.2% -- replaced with /var/log/ceph/cluster2-client.mirror.3.94215.log.gz 2026-02-22T22:10:37.758 INFO:teuthology.orchestra.run.trial104.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79269.log.gz 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.54899.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88299.log 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.54899.log.gz 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51007.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58000.log 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51007.log.gz 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88299.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92537.log 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88299.log.gz 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65976.log/var/log/ceph/cluster1-client.admin.58000.log: 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47347.log 2026-02-22T22:10:37.759 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.58000.log.gz 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92537.log.gz 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.65976.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51750.log 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47347.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65976.log.gz 0.0% 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.47347.log.gz 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57849.log 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49145.log 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51750.log.gz 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57849.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86288.log 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.49145.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.49145.log.gz 2026-02-22T22:10:37.760 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81158.log 2026-02-22T22:10:37.761 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86288.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75647.log 2026-02-22T22:10:37.761 INFO:teuthology.orchestra.run.trial104.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.86288.log.gz 2026-02-22T22:10:37.761 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78003.log 2026-02-22T22:10:37.761 INFO:teuthology.orchestra.run.trial104.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57849.log.gz 2026-02-22T22:10:37.761 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.81158.log.gz 2026-02-22T22:10:37.761 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75647.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60315.log 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78003.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.75647.log.gz 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56278.log 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.60315.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60684.log 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78003.log.gz 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.60315.log.gz 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67337.log 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.56278.log.gz 2026-02-22T22:10:37.762 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5/var/log/ceph/cluster1-client.admin.60684.log: --verbose -- /var/log/ceph/cluster2-client.admin.59212.log 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67337.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63847.log 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.60684.log.gz 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.67337.log.gz 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.59212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82195.log 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63847.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.59212.log.gz 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68148.log 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.63847.log.gz 2026-02-22T22:10:37.763 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82195.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89513.log 2026-02-22T22:10:37.764 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82195.log.gz 2026-02-22T22:10:37.764 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68148.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93203.log 2026-02-22T22:10:37.764 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.68148.log.gz 2026-02-22T22:10:37.764 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89513.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80774.log 2026-02-22T22:10:37.764 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89513.log.gz 2026-02-22T22:10:37.764 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96454.log 2026-02-22T22:10:37.764 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93203.log.gz 2026-02-22T22:10:37.765 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80774.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77447.log 2026-02-22T22:10:37.765 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80774.log.gz 2026-02-22T22:10:37.765 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.96454.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85460.log 2026-02-22T22:10:37.765 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77447.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88130.log 2026-02-22T22:10:37.765 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.96454.log.gz 2026-02-22T22:10:37.765 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77447.log.gz 2026-02-22T22:10:37.765 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.95702.log 2026-02-22T22:10:37.766 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85460.log: /var/log/ceph/cluster2-client.mirror.88130.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76984.log 2026-02-22T22:10:37.766 INFO:teuthology.orchestra.run.trial104.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.88130.log.gz 2026-02-22T22:10:37.766 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.95702.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.90245.log 2026-02-22T22:10:37.766 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.76984.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75404.log 2026-02-22T22:10:37.766 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.90245.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85460.log.gz 2026-02-22T22:10:37.766 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61240.log 2026-02-22T22:10:37.766 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.90245.log.gz 2026-02-22T22:10:37.767 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64360.log 2026-02-22T22:10:37.767 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.75404.log.gz 2026-02-22T22:10:37.767 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61240.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55904.log 2026-02-22T22:10:37.767 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.61240.log.gz 2026-02-22T22:10:37.767 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64360.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65186.log 2026-02-22T22:10:37.767 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.1.95702.log.gz 2026-02-22T22:10:37.768 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.64360.log.gz 2026-02-22T22:10:37.768 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.55904.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69761.log 2026-02-22T22:10:37.768 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.65186.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59362.log 2026-02-22T22:10:37.768 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.65186.log.gz 2026-02-22T22:10:37.768 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.55904.log.gz 2026-02-22T22:10:37.768 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.69761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51834.log 2026-02-22T22:10:37.768 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.69761.log.gz 2026-02-22T22:10:37.769 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59362.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.59362.log.gz 2026-02-22T22:10:37.769 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91544.log 2026-02-22T22:10:37.769 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51834.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79650.log 2026-02-22T22:10:37.769 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.51834.log.gz 2026-02-22T22:10:37.769 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91544.log: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.76984.log.gz 2026-02-22T22:10:37.769 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43907.log 2026-02-22T22:10:37.770 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91544.log.gz 2026-02-22T22:10:37.770 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85679.log 2026-02-22T22:10:37.770 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79650.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79650.log.gz/var/log/ceph/cluster2-client.admin.43907.log: 2026-02-22T22:10:37.770 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71054.log 2026-02-22T22:10:37.770 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43907.log.gz 2026-02-22T22:10:37.770 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85679.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91907.log 2026-02-22T22:10:37.770 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85679.log.gz 2026-02-22T22:10:37.771 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.71054.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49699.log 2026-02-22T22:10:37.771 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.71054.log.gz 2026-02-22T22:10:37.771 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78301.log 2026-02-22T22:10:37.771 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49699.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48077.log 2026-02-22T22:10:37.771 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60136.log 2026-02-22T22:10:37.771 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.49699.log.gz 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.48077.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82845.log 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48077.log.gz 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr: 89.3%/var/log/ceph/cluster2-client.admin.60136.log: -- replaced with /var/log/ceph/cluster2-client.mirror.78301.log.gz 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81934.log 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.60136.log.gz 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91907.log.gz 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53637.log 2026-02-22T22:10:37.772 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73627.log 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81934.log: /var/log/ceph/cluster2-client.mirror.53637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53736.log 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.81934.log.gz 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.53637.log.gz 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73627.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60164.log 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.73627.log.gz 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53736.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65040.log 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr: 80.9% -- replaced with /var/log/ceph/cluster2-client.mirror.53736.log.gz 2026-02-22T22:10:37.773 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.60164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88739.log 2026-02-22T22:10:37.774 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65040.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93167.log 2026-02-22T22:10:37.774 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.65040.log.gz 2026-02-22T22:10:37.774 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.60164.log.gz 2026-02-22T22:10:37.774 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88739.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70348.log 2026-02-22T22:10:37.774 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93167.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61751.log 2026-02-22T22:10:37.774 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70348.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91758.log 2026-02-22T22:10:37.775 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93167.log.gz 2026-02-22T22:10:37.775 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.70348.log.gz 2026-02-22T22:10:37.775 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61751.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.88739.log.gz 2026-02-22T22:10:37.775 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55315.log 2026-02-22T22:10:37.775 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.61751.log.gz 2026-02-22T22:10:37.775 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91758.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91214.log 2026-02-22T22:10:37.775 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55315.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91567.log 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.82845.log.gz 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr: 87.5%/var/log/ceph/cluster2-client.mirror.91214.log: -- replaced with /var/log/ceph/cluster2-client.mirror.55315.log.gz 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50474.log 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91214.log.gz 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91567.log: 92.9% -- replaced with /var/log/ceph/cluster2-client.mirror.91758.log.gz 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80378.log 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91567.log.gz 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50474.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92084.log 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50474.log.gz 2026-02-22T22:10:37.776 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89381.log 2026-02-22T22:10:37.777 INFO:teuthology.orchestra.run.trial104.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.80378.log.gz 2026-02-22T22:10:37.777 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92084.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93568.log 2026-02-22T22:10:37.777 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89381.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.92084.log.gz 2026-02-22T22:10:37.777 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74214.log 2026-02-22T22:10:37.777 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89381.log.gz 2026-02-22T22:10:37.777 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57650.log 2026-02-22T22:10:37.777 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.93568.log.gz 2026-02-22T22:10:37.778 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74214.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65068.log 2026-02-22T22:10:37.778 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.74214.log.gz 2026-02-22T22:10:37.778 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76731.log 2026-02-22T22:10:37.778 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57650.log.gz 2026-02-22T22:10:37.778 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.65068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92634.log 2026-02-22T22:10:37.778 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.65068.log.gz 2026-02-22T22:10:37.778 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76731.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67008.log 2026-02-22T22:10:37.779 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92634.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.76731.log.gz 2026-02-22T22:10:37.779 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68852.log 2026-02-22T22:10:37.779 INFO:teuthology.orchestra.run.trial048.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-22T22:10:37.779 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92634.log.gz 2026-02-22T22:10:37.779 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67008.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77006.log 2026-02-22T22:10:37.779 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.67008.log.gz 2026-02-22T22:10:37.779 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.68852.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53275.log 2026-02-22T22:10:37.780 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68852.log.gz 2026-02-22T22:10:37.780 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77006.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95193.log 2026-02-22T22:10:37.780 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56247.log 2026-02-22T22:10:37.780 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53275.log: /var/log/ceph/cluster2-client.mirror.95193.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81908.log 2026-02-22T22:10:37.780 INFO:teuthology.orchestra.run.trial104.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.53275.log.gz 2026-02-22T22:10:37.780 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56247.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71026.log 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81908.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.56247.log.gz 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49554.log 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.81908.log.gz 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71026.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57246.log 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.71026.log.gz 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49554.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48966.log 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49554.log.gz 2026-02-22T22:10:37.781 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.57246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60343.log 2026-02-22T22:10:37.782 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.57246.log.gz 2026-02-22T22:10:37.782 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.48966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67524.log 2026-02-22T22:10:37.782 INFO:teuthology.orchestra.run.trial104.stderr: 56.2% -- replaced with /var/log/ceph/cluster1-client.mirror.48966.log.gz 2026-02-22T22:10:37.782 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.60343.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89147.log 2026-02-22T22:10:37.782 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67524.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.60343.log.gz 2026-02-22T22:10:37.782 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64227.log 2026-02-22T22:10:37.782 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.67524.log.gz 2026-02-22T22:10:37.783 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89147.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61482.log 2026-02-22T22:10:37.783 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.64227.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89147.log.gz 2026-02-22T22:10:37.783 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83510.log 2026-02-22T22:10:37.783 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64227.log.gz 2026-02-22T22:10:37.783 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64085.log 2026-02-22T22:10:37.783 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.61482.log.gz 2026-02-22T22:10:37.783 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.83510.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83447.log 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77006.log.gz 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64085.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76116.log 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.83447.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.64085.log.gz 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76487.log 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.admin.83510.log.gz 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.83447.log.gz 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.76116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46839.log 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.76487.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.76116.log.gz 2026-02-22T22:10:37.784 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76375.log 2026-02-22T22:10:37.785 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.46839.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46839.log.gz 2026-02-22T22:10:37.785 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92750.log 2026-02-22T22:10:37.785 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89303.log 2026-02-22T22:10:37.785 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76375.log: /var/log/ceph/cluster2-client.mirror.92750.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.76375.log.gz 2026-02-22T22:10:37.785 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77238.log 2026-02-22T22:10:37.785 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92750.log.gz 2026-02-22T22:10:37.786 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.89303.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52666.log 2026-02-22T22:10:37.786 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.89303.log.gz 2026-02-22T22:10:37.786 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54371.log 2026-02-22T22:10:37.786 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52666.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90696.log 2026-02-22T22:10:37.786 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77238.log.gz 2026-02-22T22:10:37.786 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.52666.log.gz 2026-02-22T22:10:37.786 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54371.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81852.log 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54371.log.gz 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71934.log 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81852.log: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.90696.log.gz 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81852.log.gz 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58060.log 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71934.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54933.log 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58060.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.71934.log.gz 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83373.log 2026-02-22T22:10:37.787 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.58060.log.gz 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.54933.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62154.log 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.83373.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.54933.log.gz 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62154.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.83373.log.gz 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54294.log 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.62154.log.gz 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.76487.log.gz 2026-02-22T22:10:37.788 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72898.log 2026-02-22T22:10:37.789 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54294.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76700.log 2026-02-22T22:10:37.789 INFO:teuthology.orchestra.run.trial048.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-22T22:10:37.789 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.54294.log.gz 2026-02-22T22:10:37.789 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.72898.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85870.log 2026-02-22T22:10:37.789 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.76700.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.72898.log.gz 2026-02-22T22:10:37.789 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59331.log 2026-02-22T22:10:37.790 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.85870.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58817.log 2026-02-22T22:10:37.790 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.76700.log.gz 2026-02-22T22:10:37.790 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59331.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65463.log 2026-02-22T22:10:37.790 INFO:teuthology.orchestra.run.trial104.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.85870.log.gz 2026-02-22T22:10:37.790 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.59331.log.gz 2026-02-22T22:10:37.790 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.58817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.86234.log 2026-02-22T22:10:37.790 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51927.log/var/log/ceph/cluster2-client.admin.65463.log: 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.86234.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.58817.log.gz 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74473.log 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.86234.log.gz 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.65463.log.gz 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51927.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.87368.log 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74473.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51927.log.gz 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61057.log 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.74473.log.gz 2026-02-22T22:10:37.791 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.87368.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75937.log 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61057.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.94111.log 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.61057.log.gz 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75937.log: gzip 19.6% -5 --verbose -- replaced with /var/log/ceph/cluster1-client.mirror.1.87368.log.gz -- /var/log/ceph/cluster1-client.mirror.59114.log 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.75937.log.gz 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.1.94111.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49388.log 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.94111.log.gz 2026-02-22T22:10:37.792 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.59114.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51657.log 2026-02-22T22:10:37.793 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59114.log.gz 2026-02-22T22:10:37.793 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49388.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56538.log 2026-02-22T22:10:37.793 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67306.log 2026-02-22T22:10:37.793 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-22T22:10:37.793 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56538.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.51657.log.gz 2026-02-22T22:10:37.793 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49388.log 2026-02-22T22:10:37.793 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.56538.log.gz 2026-02-22T22:10:37.794 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose --/var/log/ceph/cluster2-client.admin.67306.log: /var/log/ceph/cluster2-client.admin.46934.log 2026-02-22T22:10:37.794 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.49388.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.67306.log.gz 2026-02-22T22:10:37.794 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94753.log 2026-02-22T22:10:37.794 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.46934.log: gzip 80.6% -- replaced with /var/log/ceph/cluster2-client.admin.46934.log.gz -5 2026-02-22T22:10:37.794 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.59698.log 2026-02-22T22:10:37.794 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94753.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84765.log 2026-02-22T22:10:37.795 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59698.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88788.log 2026-02-22T22:10:37.795 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.59698.log.gz 2026-02-22T22:10:37.795 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51393.log 2026-02-22T22:10:37.795 INFO:teuthology.orchestra.run.trial104.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49388.log.gz 2026-02-22T22:10:37.795 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88788.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64022.log 2026-02-22T22:10:37.795 INFO:teuthology.orchestra.run.trial104.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.84765.log.gz 2026-02-22T22:10:37.795 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51393.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60995.log 2026-02-22T22:10:37.796 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51393.log.gz 2026-02-22T22:10:37.796 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64022.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85327.log 2026-02-22T22:10:37.796 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.64022.log.gz 2026-02-22T22:10:37.796 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60995.log: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88788.log.gz 2026-02-22T22:10:37.796 INFO:teuthology.orchestra.run.trial104.stderr: 88.4% -- replaced with /var/log/ceph/cluster1-client.admin.60995.log.gz 2026-02-22T22:10:37.798 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97490.log 2026-02-22T22:10:37.798 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.94753.log.gz 2026-02-22T22:10:37.798 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.85327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50314.log 2026-02-22T22:10:37.798 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.85327.log.gz 2026-02-22T22:10:37.799 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97490.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97490.log.gz 2026-02-22T22:10:37.799 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75344.log 2026-02-22T22:10:37.799 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.50314.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85396.log 2026-02-22T22:10:37.799 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.75344.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49388.log.gz 2026-02-22T22:10:37.799 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70286.log 2026-02-22T22:10:37.799 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.75344.log.gz 2026-02-22T22:10:37.799 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85396.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60255.log 2026-02-22T22:10:37.800 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70286.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80645.log 2026-02-22T22:10:37.800 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85396.log.gz 2026-02-22T22:10:37.800 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50314.log.gz 2026-02-22T22:10:37.800 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.70286.log.gz 2026-02-22T22:10:37.800 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60255.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87866.log 2026-02-22T22:10:37.800 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.60255.log.gz 2026-02-22T22:10:37.800 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80645.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80489.log 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87866.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47345.log 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.87866.log.gz 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80489.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76845.log 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.80489.log.gz 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92418.log 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47345.log.gz 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.76845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80978.log 2026-02-22T22:10:37.801 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92418.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52763.log 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.76845.log.gz 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.92418.log.gz 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57018.log 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52763.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73352.log 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80978.log.gz 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.52763.log.gz 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.57018.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79018.log 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.73352.log: gzip 88.1% -5 --verbose -- replaced with /var/log/ceph/cluster2-client.admin.57018.log.gz -- 2026-02-22T22:10:37.802 INFO:teuthology.orchestra.run.trial104.stderr: /var/log/ceph/cluster1-client.mirror.94412.log 2026-02-22T22:10:37.803 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73352.log.gz 2026-02-22T22:10:37.803 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79018.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.94213.log 2026-02-22T22:10:37.803 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79018.log.gz 2026-02-22T22:10:37.803 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65522.log 2026-02-22T22:10:37.803 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.94412.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94412.log.gz 2026-02-22T22:10:37.803 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.2.94213.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87762.log 2026-02-22T22:10:37.803 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.65522.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81628.log 2026-02-22T22:10:37.804 INFO:teuthology.orchestra.run.trial104.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.80645.log.gz 2026-02-22T22:10:37.804 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.65522.log.gz 2026-02-22T22:10:37.804 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5/var/log/ceph/cluster1-client.mirror.87762.log: --verbose -- /var/log/ceph/cluster1-client.mirror.80512.log 2026-02-22T22:10:37.804 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81628.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87762.log.gz 2026-02-22T22:10:37.804 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81628.log.gz 2026-02-22T22:10:37.804 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97195.log 2026-02-22T22:10:37.804 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84876.log 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97195.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97195.log.gz 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91933.log 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50022.log 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.2.94213.log.gz 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49431.log 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80512.log: /var/log/ceph/cluster2-client.mirror.91933.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.80512.log.gz 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50022.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84876.log.gz 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91885.log 2026-02-22T22:10:37.805 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.91933.log.gz 2026-02-22T22:10:37.806 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50022.log.gz 2026-02-22T22:10:37.806 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49431.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88028.log 2026-02-22T22:10:37.806 INFO:teuthology.orchestra.run.trial104.stderr: 83.1% -- replaced with /var/log/ceph/cluster2-client.mirror.49431.log.gz 2026-02-22T22:10:37.807 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89240.log 2026-02-22T22:10:37.807 INFO:teuthology.orchestra.run.trial104.stderr: 92.6% -- replaced with /var/log/ceph/cluster2-client.mirror.91885.log.gz 2026-02-22T22:10:37.807 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.88028.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92456.log 2026-02-22T22:10:37.807 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.88028.log.gz 2026-02-22T22:10:37.807 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89240.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79778.log 2026-02-22T22:10:37.808 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89240.log.gz 2026-02-22T22:10:37.808 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92456.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47979.log 2026-02-22T22:10:37.808 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92383.log 2026-02-22T22:10:37.808 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79778.log.gz 2026-02-22T22:10:37.808 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.47979.log: gzip -5 --verbose 0.0% -- /var/log/ceph/cluster1-client.admin.72600.log -- replaced with /var/log/ceph/cluster1-client.admin.47979.log.gz 2026-02-22T22:10:37.808 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.808 INFO:teuthology.orchestra.run.trial104.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92456.log.gz 2026-02-22T22:10:37.809 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92383.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77195.log 2026-02-22T22:10:37.809 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72600.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.92383.log.gz 2026-02-22T22:10:37.809 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72471.log 2026-02-22T22:10:37.809 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.72600.log.gz 2026-02-22T22:10:37.809 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92572.log 2026-02-22T22:10:37.809 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77195.log: /var/log/ceph/cluster2-client.mirror.72471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80999.log 2026-02-22T22:10:37.809 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.72471.log.gz 2026-02-22T22:10:37.810 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86553.log 2026-02-22T22:10:37.810 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92572.log.gz 2026-02-22T22:10:37.810 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80999.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94732.log 2026-02-22T22:10:37.810 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80999.log.gz 2026-02-22T22:10:37.810 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.86553.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73970.log 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94732.log: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.86553.log.gz 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76085.log 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.94732.log.gz 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.73970.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50700.log 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.73970.log.gz 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.76085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77282.log 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.76085.log.gz 2026-02-22T22:10:37.811 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50700.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82172.log 2026-02-22T22:10:37.812 INFO:teuthology.orchestra.run.trial104.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50700.log.gz 2026-02-22T22:10:37.812 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77282.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75966.log 2026-02-22T22:10:37.812 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82172.log: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.82172.log.gz 2026-02-22T22:10:37.812 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74533.log 2026-02-22T22:10:37.812 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49000.log 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.75966.log.gz 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74533.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.77195.log.gz 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75226.log 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.74533.log.gz 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.49000.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49307.log 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49000.log.gz 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75226.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58514.log 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49307.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.75226.log.gz 2026-02-22T22:10:37.813 INFO:teuthology.orchestra.run.trial104.stderr: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49307.log.gz 2026-02-22T22:10:37.814 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61885.log 2026-02-22T22:10:37.814 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78130.log 2026-02-22T22:10:37.814 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58514.log: /var/log/ceph/cluster1-client.mirror.61885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60602.log 2026-02-22T22:10:37.814 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.58514.log.gz 2026-02-22T22:10:37.814 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.61885.log.gz 2026-02-22T22:10:37.814 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78130.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74741.log 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.60602.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89358.log 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr: 90.3% -- replaced with /var/log/ceph/cluster1-client.mirror.78130.log.gz 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66503.log 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.60602.log.gz 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89358.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.74741.log.gz 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56567.log 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.66503.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59243.log 2026-02-22T22:10:37.815 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66503.log.gz 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.56567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56307.log 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89358.log.gz 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.56567.log.gz 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr:gzip/var/log/ceph/cluster2-client.admin.59243.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.44001.log 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.56307.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.59243.log.gzgzip 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91359.log 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.56307.log.gz 2026-02-22T22:10:37.816 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.44001.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.44001.log.gz 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71241.log 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54259.log 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.91359.log.gz 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.71241.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76404.log 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr: 93.4% -- replaced with /var/log/ceph/cluster2-client.mirror.77282.log.gz 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62494.log 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.71241.log.gz 2026-02-22T22:10:37.817 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54259.log.gz 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.76404.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69223.log 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.76404.log.gz 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62494.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50635.log 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.69223.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.62494.log.gz 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78826.log 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.50635.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69223.log.gz 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58996.log 2026-02-22T22:10:37.818 INFO:teuthology.orchestra.run.trial104.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.50635.log.gz 2026-02-22T22:10:37.819 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78826.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88834.log 2026-02-22T22:10:37.819 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.78826.log.gz 2026-02-22T22:10:37.819 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.58996.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81203.log 2026-02-22T22:10:37.819 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88834.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.58996.log.gz 2026-02-22T22:10:37.819 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46759.log 2026-02-22T22:10:37.819 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.88834.log.gz 2026-02-22T22:10:37.819 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84854.log 2026-02-22T22:10:37.820 INFO:teuthology.orchestra.run.trial104.stderr: 54.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81203.log.gz 2026-02-22T22:10:37.820 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.46759.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59533.log 2026-02-22T22:10:37.820 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46759.log.gz 2026-02-22T22:10:37.820 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53989.log 2026-02-22T22:10:37.820 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.59533.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49119.log 2026-02-22T22:10:37.820 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84854.log.gz 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59533.log.gz 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53989.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63527.log 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53989.log.gz 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr:gzip/var/log/ceph/cluster2-client.admin.49119.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.48126.log 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.49119.log.gz 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63527.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63345.log 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.63527.log.gz 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.48126.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48126.log.gz 2026-02-22T22:10:37.821 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68919.log 2026-02-22T22:10:37.822 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63345.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85807.log 2026-02-22T22:10:37.822 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68919.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55392.log 2026-02-22T22:10:37.822 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.63345.log.gz 2026-02-22T22:10:37.822 INFO:teuthology.orchestra.run.trial104.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.68919.log.gz 2026-02-22T22:10:37.822 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85807.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74123.log 2026-02-22T22:10:37.822 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85807.log.gz 2026-02-22T22:10:37.823 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78346.log 2026-02-22T22:10:37.823 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.74123.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55392.log.gz 2026-02-22T22:10:37.823 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50633.log 2026-02-22T22:10:37.823 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78346.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58088.log 2026-02-22T22:10:37.823 INFO:teuthology.orchestra.run.trial104.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.74123.log.gz 2026-02-22T22:10:37.823 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.50633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78277.log 2026-02-22T22:10:37.823 INFO:teuthology.orchestra.run.trial104.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78346.log.gz 2026-02-22T22:10:37.824 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.58088.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54014.log 2026-02-22T22:10:37.824 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78277.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.58088.log.gz 2026-02-22T22:10:37.824 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85052.log 2026-02-22T22:10:37.824 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78277.log.gz 2026-02-22T22:10:37.824 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54014.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79337.log 2026-02-22T22:10:37.824 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54014.log.gz 2026-02-22T22:10:37.824 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49379.log 2026-02-22T22:10:37.825 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79337.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67821.log 2026-02-22T22:10:37.825 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79337.log.gz 2026-02-22T22:10:37.825 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85052.log.gz 2026-02-22T22:10:37.825 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.49379.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49379.log.gz 2026-02-22T22:10:37.825 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59786.log 2026-02-22T22:10:37.825 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.67821.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93786.log 2026-02-22T22:10:37.825 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67821.log.gz 2026-02-22T22:10:37.826 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59786.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81367.log 2026-02-22T22:10:37.826 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93786.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.59786.log.gz 2026-02-22T22:10:37.826 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90920.log 2026-02-22T22:10:37.826 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81367.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.93786.log.gz 2026-02-22T22:10:37.826 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91016.log 2026-02-22T22:10:37.826 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81367.log.gz 2026-02-22T22:10:37.826 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90920.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68769.log 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91016.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89272.log 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91016.log.gz 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.68769.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86175.log 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90920.log.gz 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.89272.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.68769.log.gz 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61271.log 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86175.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.89272.log.gz 2026-02-22T22:10:37.827 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53244.log 2026-02-22T22:10:37.828 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.86175.log.gz 2026-02-22T22:10:37.828 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66223.log 2026-02-22T22:10:37.828 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.61271.log.gz 2026-02-22T22:10:37.828 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53098.log 2026-02-22T22:10:37.828 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66223.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.53244.log.gz 2026-02-22T22:10:37.828 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49324.log 2026-02-22T22:10:37.828 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.66223.log.gz 2026-02-22T22:10:37.829 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53098.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59079.log 2026-02-22T22:10:37.829 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49324.log: 64.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49324.log.gz 2026-02-22T22:10:37.829 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.53098.log.gz 2026-02-22T22:10:37.829 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68532.log 2026-02-22T22:10:37.829 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.59079.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93145.log 2026-02-22T22:10:37.829 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59079.log.gz 2026-02-22T22:10:37.829 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68532.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56021.log 2026-02-22T22:10:37.830 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.68532.log.gz 2026-02-22T22:10:37.830 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.93145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66976.log 2026-02-22T22:10:37.830 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56021.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58937.log 2026-02-22T22:10:37.830 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.66976.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.56021.log.gz 2026-02-22T22:10:37.830 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63617.log 2026-02-22T22:10:37.830 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66976.log.gz 2026-02-22T22:10:37.830 INFO:teuthology.orchestra.run.trial104.stderr: 92.5% -- replaced with /var/log/ceph/cluster1-client.mirror.93145.log.gz 2026-02-22T22:10:37.831 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58937.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73655.log 2026-02-22T22:10:37.831 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63617.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94153.log 2026-02-22T22:10:37.831 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.58937.log.gz 2026-02-22T22:10:37.831 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.73655.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.63617.log.gz 2026-02-22T22:10:37.831 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55750.log 2026-02-22T22:10:37.831 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.73655.log.gz 2026-02-22T22:10:37.831 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.94153.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55633.log 2026-02-22T22:10:37.832 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94153.log.gz 2026-02-22T22:10:37.832 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.55750.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85030.log 2026-02-22T22:10:37.832 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.55750.log.gz 2026-02-22T22:10:37.832 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55633.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69559.log 2026-02-22T22:10:37.832 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85030.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77331.log 2026-02-22T22:10:37.832 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55633.log.gz 2026-02-22T22:10:37.832 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69559.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56503.log 2026-02-22T22:10:37.833 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85030.log.gz 2026-02-22T22:10:37.833 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.69559.log.gz 2026-02-22T22:10:37.833 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77331.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78206.log 2026-02-22T22:10:37.833 INFO:teuthology.orchestra.run.trial104.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77331.log.gz 2026-02-22T22:10:37.833 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86587.log 2026-02-22T22:10:37.833 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56503.log: /var/log/ceph/cluster1-client.mirror.78206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79092.log 2026-02-22T22:10:37.833 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.56503.log.gz 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.86587.log: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78206.log.gz 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58454.log 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.86587.log.gz 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79092.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91051.log 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.79092.log.gz 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58454.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94275.log 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.58454.log.gz 2026-02-22T22:10:37.834 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78176.log 2026-02-22T22:10:37.835 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91051.log.gz 2026-02-22T22:10:37.835 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.94275.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90867.log 2026-02-22T22:10:37.835 INFO:teuthology.orchestra.run.trial104.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94275.log.gz 2026-02-22T22:10:37.835 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87904.log 2026-02-22T22:10:37.835 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90867.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96508.log 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85228.log 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.78176.log.gz 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.96508.log: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.87904.log.gz 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.95060.log 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.96508.log.gz 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.85228.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87887.log 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85228.log.gz 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.90867.log.gz 2026-02-22T22:10:37.836 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.95060.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75131.log 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.95060.log.gz 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63136.log 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr: 58.1% -- replaced with /var/log/ceph/cluster2-client.mirror.87887.log.gz 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75131.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56050.log 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.2.50633.log.gz 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63136.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.75131.log.gz 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47465.log 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.63136.log.gz 2026-02-22T22:10:37.837 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.56050.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86618.log 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.56050.log.gz 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47465.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73324.log 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47465.log.gz 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.86618.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85206.log 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73324.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.86618.log.gz 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73260.log 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.73324.log.gz 2026-02-22T22:10:37.838 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63991.log 2026-02-22T22:10:37.839 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85206.log: /var/log/ceph/cluster1-client.admin.73260.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58625.log 2026-02-22T22:10:37.839 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.73260.log.gz 2026-02-22T22:10:37.839 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63991.log: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85206.log.gz 2026-02-22T22:10:37.839 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55273.log 2026-02-22T22:10:37.839 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.63991.log.gz 2026-02-22T22:10:37.839 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.58625.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52011.log 2026-02-22T22:10:37.839 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.58625.log.gz 2026-02-22T22:10:37.840 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88913.log 2026-02-22T22:10:37.840 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55273.log.gz 2026-02-22T22:10:37.840 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-22T22:10:37.840 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.52011.log.gz 2026-02-22T22:10:37.840 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.88913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73085.log 2026-02-22T22:10:37.840 INFO:teuthology.orchestra.run.trial104.stderr: 86.4% -- replaced with /var/log/ceph/cluster1-client.mirror.88913.log.gz 2026-02-22T22:10:37.840 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71626.log 2026-02-22T22:10:37.841 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73085.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85417.log 2026-02-22T22:10:37.841 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.73085.log.gz 2026-02-22T22:10:37.841 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.71626.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92292.log 2026-02-22T22:10:37.841 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.71626.log.gz 2026-02-22T22:10:37.841 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83342.log 2026-02-22T22:10:37.841 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92292.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72741.log 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.83342.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85417.log.gz 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92292.log.gz 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73502.log 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.83342.log.gz 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72741.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68096.log 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.72741.log.gz 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68182.log 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.73502.log.gz 2026-02-22T22:10:37.842 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.68096.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62927.log 2026-02-22T22:10:37.843 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51688.log 2026-02-22T22:10:37.843 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.68096.log.gz 2026-02-22T22:10:37.843 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.68182.log.gz 2026-02-22T22:10:37.843 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61203.log 2026-02-22T22:10:37.843 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62927.log: /var/log/ceph/cluster1-client.admin.51688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75766.log 2026-02-22T22:10:37.843 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.62927.log.gz 2026-02-22T22:10:37.844 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61203.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51688.log.gz 2026-02-22T22:10:37.844 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86800.log 2026-02-22T22:10:37.844 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.61203.log.gz 2026-02-22T22:10:37.844 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.75766.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72281.log 2026-02-22T22:10:37.844 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86800.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.49431.log 2026-02-22T22:10:37.844 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.75766.log.gz 2026-02-22T22:10:37.844 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72281.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77638.log 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.72281.log.gz 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.49431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85772.log 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.86800.log.gz 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.77638.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57126.log 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85772.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.77638.log.gz 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51039.log 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85772.log.gz 2026-02-22T22:10:37.845 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.57126.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64479.log 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51039.log: 90.1%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.57126.log.gz 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78613.log 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr: 91.8% -- replaced with /var/log/ceph/cluster1-client.mirror.1.49431.log.gz 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.51039.log.gz/var/log/ceph/cluster1-client.admin.64479.log: 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95702.log 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr: 88.0%/var/log/ceph/cluster1-client.mirror.78613.log: -- replaced with /var/log/ceph/cluster1-client.admin.64479.log.gz 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91439.log 2026-02-22T22:10:37.846 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.78613.log.gz 2026-02-22T22:10:37.847 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.95702.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59391.log 2026-02-22T22:10:37.847 INFO:teuthology.orchestra.run.trial104.stderr: 86.5% -- replaced with /var/log/ceph/cluster2-client.mirror.95702.log.gz 2026-02-22T22:10:37.847 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91439.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70467.log 2026-02-22T22:10:37.847 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.91439.log.gz 2026-02-22T22:10:37.847 INFO:teuthology.orchestra.run.trial104.stderr:gzip/var/log/ceph/cluster2-client.admin.59391.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.82662.log 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70467.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.59391.log.gz 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59976.log 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.70467.log.gz 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.82662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70250.log 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr: 80.2% -- replaced with /var/log/ceph/cluster1-client.admin.82662.log.gz 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.59976.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80354.log 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.59976.log.gz 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.70250.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57748.log 2026-02-22T22:10:37.848 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.70250.log.gz 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80354.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66192.log 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.80354.log.gz 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.57748.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76921.log 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.57748.log.gz 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66192.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56808.log 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76921.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.66192.log.gz 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88581.log 2026-02-22T22:10:37.849 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.76921.log.gz 2026-02-22T22:10:37.850 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56808.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90769.log 2026-02-22T22:10:37.850 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56808.log.gz 2026-02-22T22:10:37.850 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88581.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85484.log 2026-02-22T22:10:37.850 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88581.log.gz 2026-02-22T22:10:37.850 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.90769.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77051.log 2026-02-22T22:10:37.850 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.90769.log.gz 2026-02-22T22:10:37.851 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85484.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96276.log 2026-02-22T22:10:37.851 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.77051.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79918.log 2026-02-22T22:10:37.851 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.96276.log: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.85484.log.gz 2026-02-22T22:10:37.851 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89434.log 2026-02-22T22:10:37.851 INFO:teuthology.orchestra.run.trial104.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.admin.96276.log.gz 2026-02-22T22:10:37.851 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.79918.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60841.log 2026-02-22T22:10:37.851 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89434.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.77051.log.gz 2026-02-22T22:10:37.852 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.79918.log.gz 2026-02-22T22:10:37.852 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81020.log 2026-02-22T22:10:37.852 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.89434.log.gz 2026-02-22T22:10:37.852 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.60841.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93382.log 2026-02-22T22:10:37.852 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81020.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.60841.log.gz 2026-02-22T22:10:37.852 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93987.log 2026-02-22T22:10:37.852 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94109.log 2026-02-22T22:10:37.853 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81020.log.gz 2026-02-22T22:10:37.853 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93987.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.93382.log.gz 2026-02-22T22:10:37.853 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50528.log 2026-02-22T22:10:37.853 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.93987.log.gz 2026-02-22T22:10:37.853 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.94109.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.87002.log 2026-02-22T22:10:37.853 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80463.log 2026-02-22T22:10:37.853 INFO:teuthology.orchestra.run.trial104.stderr: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.94109.log.gz 2026-02-22T22:10:37.854 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.87002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89963.log 2026-02-22T22:10:37.854 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.87002.log.gz 2026-02-22T22:10:37.854 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81266.log 2026-02-22T22:10:37.854 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87523.log 2026-02-22T22:10:37.854 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81266.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75872.log 2026-02-22T22:10:37.855 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51070.log 2026-02-22T22:10:37.855 INFO:teuthology.orchestra.run.trial104.stderr: 88.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87523.log.gz 2026-02-22T22:10:37.855 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.81266.log.gz 2026-02-22T22:10:37.855 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.75872.log: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89963.log.gz 2026-02-22T22:10:37.855 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.90738.log 2026-02-22T22:10:37.855 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.75872.log.gz 2026-02-22T22:10:37.855 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51070.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71695.log 2026-02-22T22:10:37.856 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.90738.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51070.log.gz 2026-02-22T22:10:37.856 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51590.log 2026-02-22T22:10:37.856 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.90738.log.gz 2026-02-22T22:10:37.856 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71695.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.90199.log 2026-02-22T22:10:37.856 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.71695.log.gz 2026-02-22T22:10:37.856 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.51590.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50294.log 2026-02-22T22:10:37.857 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.90199.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81220.log 2026-02-22T22:10:37.857 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50294.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62833.log 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81220.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69164.log 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.51590.log.gz 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.62833.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80463.log.gz 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93511.log 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62833.log.gz 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.81220.log.gz 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.69164.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72372.log 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50294.log.gz 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93511.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.69164.log.gz 2026-02-22T22:10:37.858 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74887.log 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.93511.log.gz 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53433.log 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.72372.log.gz 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.74887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56180.log 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.74887.log.gz 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76025.log 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 93.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95193.log.gz 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 88.1%/var/log/ceph/cluster2-client.mirror.56180.log: -- replaced with /var/log/ceph/cluster1-client.admin.53433.log.gz 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47048.log 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56180.log.gz 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.76025.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71420.log 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71903.log 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47048.log.gz 2026-02-22T22:10:37.859 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.76025.log.gz 2026-02-22T22:10:37.860 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.71420.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87410.log 2026-02-22T22:10:37.860 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.71420.log.gz 2026-02-22T22:10:37.860 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63195.log 2026-02-22T22:10:37.860 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.71903.log.gz 2026-02-22T22:10:37.860 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87410.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64419.log 2026-02-22T22:10:37.860 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.87410.log.gz 2026-02-22T22:10:37.860 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.63195.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96580.log 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87789.log 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.63195.log.gz 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.64419.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91521.log 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.96580.log: 29.5% -- replaced with /var/log/ceph/cluster2-client.admin.96580.log.gz 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75557.log 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.64419.log.gz 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87789.log.gz 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91521.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60105.log 2026-02-22T22:10:37.861 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.91521.log.gz 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.75557.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69341.log 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.60105.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.92052.log 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.75557.log.gz 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.60105.log.gz 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.69341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72870.log 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.69341.log.gz 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.92052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85261.log 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.92052.log.gz 2026-02-22T22:10:37.862 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72870.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.87472.log 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.72870.log.gz 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.85261.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59568.log 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.85261.log.gz 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.87472.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90474.log 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.87472.log.gz 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.59568.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82385.log 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.59568.log.gz 2026-02-22T22:10:37.863 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57049.log 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90474.log: /var/log/ceph/cluster2-client.mirror.82385.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73474.log 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90474.log.gz 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82385.log.gz 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.57049.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63073.log 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.87367.log 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.57049.log.gz 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73474.log: /var/log/ceph/cluster1-client.admin.63073.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56731.log 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.87367.log: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.63073.log.gz 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose 88.5% -- /var/log/ceph/cluster2-client.mirror.88427.log -- replaced with /var/log/ceph/cluster2-client.admin.73474.log.gz 2026-02-22T22:10:37.864 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.865 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56731.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90594.log 2026-02-22T22:10:37.865 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56731.log.gz 2026-02-22T22:10:37.865 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88427.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76661.log 2026-02-22T22:10:37.865 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88427.log.gz 2026-02-22T22:10:37.865 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51435.log 2026-02-22T22:10:37.865 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90594.log: /var/log/ceph/cluster1-client.admin.76661.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82679.log 2026-02-22T22:10:37.865 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90594.log.gz 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68501.log 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.76661.log.gz 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51435.log.gz 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82679.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87053.log 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr: 63.3% -- replaced with /var/log/ceph/cluster1-client.mirror.82679.log.gz 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68501.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65434.log 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87053.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51134.log 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.68501.log.gz 2026-02-22T22:10:37.866 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65434.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92865.log 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.65434.log.gz 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.51134.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58758.log 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92865.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65343.log 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51134.log.gz 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.mirror.92865.log.gz 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58758.log: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.87053.log.gz 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64889.log 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.58758.log.gz 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.65343.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74643.log 2026-02-22T22:10:37.867 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.64889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80751.log 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74643.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.65343.log.gz 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr: 90.0%/var/log/ceph/cluster2-client.mirror.80751.log: -- replaced with /var/log/ceph/cluster1-client.mirror.64889.log.gz 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.74643.log.gz 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80325.log 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5/var/log/ceph/cluster2-osd.1.log: --verbose -- /var/log/ceph/cluster1-client.admin.62608.log 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80325.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63648.log 2026-02-22T22:10:37.868 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.80325.log.gz 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62608.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54166.log 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.62608.log.gz 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63648.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53690.log 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54166.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63782.log 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.63648.log.gz 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54166.log.gz 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.53690.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90899.log 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.53690.log.gz 2026-02-22T22:10:37.869 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.63782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50259.log 2026-02-22T22:10:37.870 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90899.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63782.log.gz 2026-02-22T22:10:37.870 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92911.log 2026-02-22T22:10:37.870 INFO:teuthology.orchestra.run.trial104.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90899.log.gz 2026-02-22T22:10:37.870 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.50259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76435.log 2026-02-22T22:10:37.870 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.50259.log.gz 2026-02-22T22:10:37.870 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82555.log 2026-02-22T22:10:37.870 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92911.log: /var/log/ceph/cluster1-client.admin.76435.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78635.log 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.92911.log.gz 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.76435.log.gz 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82555.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89338.log 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82555.log.gz 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78635.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82010.log 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89338.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69195.log 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82010.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80266.log 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78635.log.gz 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82010.log.gz 2026-02-22T22:10:37.871 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.69195.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53329.log 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80751.log.gz 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80266.log: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.69195.log.gz 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47346.log 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80266.log.gz 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91148.log 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89338.log.gz 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47346.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65315.log 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53329.log.gz 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47346.log.gz 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91148.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62038.log 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91148.log.gz 2026-02-22T22:10:37.872 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65315.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88111.log 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.62038.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.65315.log.gzgzip 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90171.log 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88111.log: 82.9% -- replaced with /var/log/ceph/cluster2-client.mirror.88111.log.gz 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74183.log 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.mirror.62038.log.gz 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70935.log 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr: 81.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90171.log.gz 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92476.log 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70935.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.74183.log.gz 2026-02-22T22:10:37.873 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71591.log 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92476.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.70935.log.gz 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76629.log 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.71591.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64051.log 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.71591.log.gz 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.76629.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81762.log 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.76629.log.gz 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57403.log 2026-02-22T22:10:37.874 INFO:teuthology.orchestra.run.trial104.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.92476.log.gz 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81762.log: gzip -5 88.0% --verbose -- -- replaced with /var/log/ceph/cluster2-client.admin.64051.log.gz /var/log/ceph/cluster2-client.mirror.51171.log 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81762.log.gz/var/log/ceph/cluster2-client.mirror.57403.log: 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91864.log 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57403.log.gz 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51171.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80201.log 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48469.log 2026-02-22T22:10:37.875 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80201.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84832.log 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.91864.log.gz 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.48469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.90200.log 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48469.log.gz 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr: 92.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51171.log.gz 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80201.log.gz 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84832.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96304.log 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.90200.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95762.log 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.90200.log.gz 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84832.log.gz 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.96304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72659.log 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.95762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61417.log 2026-02-22T22:10:37.876 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.72659.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57438.log 2026-02-22T22:10:37.877 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.61417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56395.log 2026-02-22T22:10:37.877 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61417.log.gz 2026-02-22T22:10:37.877 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.72659.log.gz 2026-02-22T22:10:37.877 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57438.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52549.log 2026-02-22T22:10:37.877 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57438.log.gz 2026-02-22T22:10:37.877 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64116.log 2026-02-22T22:10:37.877 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56395.log: /var/log/ceph/cluster1-client.admin.52549.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53914.log 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64116.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82443.log 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52549.log.gz 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.64116.log.gz 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.53914.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53914.log.gz 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91498.log 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.56395.log.gz 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82443.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62397.log 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.82443.log.gz 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91476.log 2026-02-22T22:10:37.878 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.91498.log.gz 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62397.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.86288.log 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91476.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.62397.log.gz 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80934.log 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91476.log.gz 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.86288.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82125.log 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80934.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89687.log 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82125.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50633.log 2026-02-22T22:10:37.879 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.96304.log.gz 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89687.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69925.log 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82125.log.gz 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50633.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94957.log 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50633.log.gz 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69925.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48616.log 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.69925.log.gz 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51570.log 2026-02-22T22:10:37.880 INFO:teuthology.orchestra.run.trial104.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89687.log.gz 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94957.log.gz 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.48616.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.79180.log 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48616.log.gz 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51570.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.86288.log.gz 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74035.log 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.79180.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70013.log 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.79180.log.gz 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74035.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.87523.log 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.74035.log.gz 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51570.log.gz 2026-02-22T22:10:37.881 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70013.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80421.log 2026-02-22T22:10:37.882 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.87523.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65605.log 2026-02-22T22:10:37.882 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.70013.log.gz 2026-02-22T22:10:37.882 INFO:teuthology.orchestra.run.trial104.stderr:gzip/var/log/ceph/cluster2-client.mirror.80421.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.71874.log 2026-02-22T22:10:37.882 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.65605.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.75075.log 2026-02-22T22:10:37.882 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.65605.log.gz 2026-02-22T22:10:37.882 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.71874.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80399.log 2026-02-22T22:10:37.882 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80421.log.gz 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.71874.log.gz 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.75075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97097.log 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80399.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48175.log 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80934.log.gz 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97097.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.75075.log.gz 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57305.log 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97097.log.gz 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63102.log 0.0% 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.48175.log.gz 2026-02-22T22:10:37.883 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57305.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67212.log 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.54476.log 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.57305.log.gz 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67212.log: 92.2% 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.3.87523.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.63102.log.gz 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61574.log 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.67212.log.gz 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.1.54476.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76145.log 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.1.54476.log.gz 2026-02-22T22:10:37.884 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.61574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85096.log 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76145.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82350.log 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.61574.log.gz 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85096.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75678.log 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.76145.log.gz 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82350.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85714.log 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85096.log.gz 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82350.log.gz 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75678.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54201.log 2026-02-22T22:10:37.885 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.75678.log.gz 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85714.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60043.log 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.85714.log.gz 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54201.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47783.log 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.54201.log.gz 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60043.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71172.log 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80399.log.gz 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47783.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69440.log 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.60043.log.gz 2026-02-22T22:10:37.886 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47783.log.gz 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.71172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70073.log 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.71172.log.gz 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.69440.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86694.log 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70073.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.69440.log.gz 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.86227.log 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.86694.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.70073.log.gz 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55990.log 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.86227.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.86694.log.gz 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76764.log 2026-02-22T22:10:37.887 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.55990.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73805.log 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76764.log: gzip 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.55990.log.gz -5 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.47734.log 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73805.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93754.log 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.76764.log.gz 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.73805.log.gz 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47734.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69650.log 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47734.log.gz 2026-02-22T22:10:37.888 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93754.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60284.log 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.69650.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.93754.log.gz 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59638.log 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.69650.log.gz 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.60284.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97048.log 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.2.86227.log.gz 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.59638.log: gzip 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.60284.log.gz -5 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.60449.log 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.59638.log.gz 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97048.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95089.log 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97048.log.gz 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.60449.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70704.log 2026-02-22T22:10:37.889 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60449.log.gz 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.95089.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93351.log 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.70704.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.95089.log.gz 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80541.log 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70704.log.gz 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93351.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71962.log 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80541.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.93351.log.gz 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92219.log 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr: 87.3%/var/log/ceph/cluster1-client.mirror.71962.log: -- replaced with /var/log/ceph/cluster1-client.mirror.80541.log.gz 2026-02-22T22:10:37.890 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81087.log 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64262.log 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.71962.log.gz 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.92219.log.gz 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81087.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78873.log 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81087.log.gz 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.64262.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90135.log 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.64262.log.gz 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78873.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75469.log 2026-02-22T22:10:37.891 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90135.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.95704.log 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78873.log.gz 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75469.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.90135.log.gz 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67793.log 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.75469.log.gz 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.95704.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50197.log 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr: 20.2% -- replaced with /var/log/ceph/cluster1-client.mirror.2.95704.log.gz 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67793.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67365.log 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.67793.log.gz 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.50197.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50495.log 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.50197.log.gz 2026-02-22T22:10:37.892 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.67365.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50495.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48915.log 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.67365.log.gz 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64762.log 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49205.log 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.48915.log.gz 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64762.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92961.log 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.64762.log.gz 2026-02-22T22:10:37.893 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95614.log 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.49205.log: 70.3%/var/log/ceph/cluster1-client.mirror.92961.log: -- replaced with /var/log/ceph/cluster2-client.admin.49205.log.gz 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93643.log 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.92961.log.gz 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.95614.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96398.log 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93643.log: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.95614.log.gz 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84986.log 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.93643.log.gz 2026-02-22T22:10:37.894 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.96398.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52238.log 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84986.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53554.log 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93087.log 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.mirror.96398.log.gz 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84986.log.gz 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.mirror.0.87367.log.gz 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.52238.log.gz 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53554.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58877.log 2026-02-22T22:10:37.895 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.53554.log.gz 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93087.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93676.log 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93087.log.gz 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92807.log 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73864.log 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93676.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.58877.log.gz 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92807.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50495.log.gz 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.93676.log.gz 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59846.log 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92807.log.gz 2026-02-22T22:10:37.896 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.73864.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85566.log 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.59846.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.73864.log.gz 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70379.log 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.59846.log.gz 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85566.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53889.log 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70379.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68985.log 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.70379.log.gz 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.53889.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61546.log 2026-02-22T22:10:37.897 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53889.log.gz 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68985.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96556.log 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5/var/log/ceph/cluster1-client.admin.61546.log: --verbose -- /var/log/ceph/cluster1-client.admin.54403.log 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.68985.log.gz 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.96556.log: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.61546.log.gz 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.96556.log.gz 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.54582.log 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.54403.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84942.log 2026-02-22T22:10:37.898 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.3.54582.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83412.log 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.54582.log.gz 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.54403.log.gz 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84942.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80158.log 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.83412.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56989.log 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.84942.log.gz 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.83412.log.gz 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80158.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77099.log 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.80158.log.gz 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56989.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66059.log 2026-02-22T22:10:37.899 INFO:teuthology.orchestra.run.trial104.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.95762.log.gz 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.56989.log.gz 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82824.log 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.66059.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60718.log 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.66059.log.gz 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82824.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86903.log 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.82824.log.gz 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.85566.log.gz 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69471.log 2026-02-22T22:10:37.900 INFO:teuthology.orchestra.run.trial104.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.77099.log.gz 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.86903.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72835.log 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.60718.log.gz 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr: 88.5%/var/log/ceph/cluster2-client.admin.69471.log: -- replaced with /var/log/ceph/cluster1-client.admin.86903.log.gz 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70773.log 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.69471.log.gz 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72835.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.95822.log 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.72835.log.gz 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67424.log 2026-02-22T22:10:37.901 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.70773.log.gz 2026-02-22T22:10:37.902 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.95822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80222.log 2026-02-22T22:10:37.902 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.67424.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81410.log 2026-02-22T22:10:37.902 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.67424.log.gz 2026-02-22T22:10:37.902 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51217.log 2026-02-22T22:10:37.902 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81410.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90643.log 2026-02-22T22:10:37.902 INFO:teuthology.orchestra.run.trial104.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.80222.log.gz 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52518.log 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.81410.log.gz 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90643.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60076.log 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.3.95822.log.gz 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.90643.log.gz 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52518.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.73173.log 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.52518.log.gz 2026-02-22T22:10:37.903 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60076.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96243.log 2026-02-22T22:10:37.904 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.60076.log.gz 2026-02-22T22:10:37.904 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.73173.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82803.log 2026-02-22T22:10:37.904 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.96243.log: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.73173.log.gz 2026-02-22T22:10:37.904 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96532.log 2026-02-22T22:10:37.904 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82803.log: 88.9% -- replaced with /var/log/ceph/cluster2-client.admin.96243.log.gz 2026-02-22T22:10:37.904 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73534.log 2026-02-22T22:10:37.904 INFO:teuthology.orchestra.run.trial104.stderr: 84.7% -- replaced with /var/log/ceph/cluster2-client.mirror.82803.log.gz 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.96532.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96532.log.gz 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79430.log 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73534.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50057.log 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.73534.log.gz 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79430.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60813.log 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79430.log.gz 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50057.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.95500.log 2026-02-22T22:10:37.905 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50057.log.gz 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.60813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51470.log 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.60813.log.gz 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.95500.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62898.log 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.95500.log.gz 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56598.log 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51470.log.gz 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77259.log 2026-02-22T22:10:37.906 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.62898.log.gz 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.56598.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92348.log 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.51217.log.gz 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.56598.log.gz 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77259.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50571.log 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92348.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82273.log 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.92348.log.gz 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77259.log.gz 2026-02-22T22:10:37.907 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.50571.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53715.log 2026-02-22T22:10:37.908 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92174.log 2026-02-22T22:10:37.908 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.82273.log.gz 2026-02-22T22:10:37.908 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.53715.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79573.log 2026-02-22T22:10:37.908 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.53715.log.gz 2026-02-22T22:10:37.908 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92174.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.94153.log 2026-02-22T22:10:37.908 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92174.log.gz 2026-02-22T22:10:37.908 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79573.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69832.log 2026-02-22T22:10:37.909 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79573.log.gz 2026-02-22T22:10:37.909 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.1.94153.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78850.log 2026-02-22T22:10:37.909 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.69832.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49490.log 2026-02-22T22:10:37.909 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.69832.log.gz 2026-02-22T22:10:37.909 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78850.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81243.log 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49490.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81606.log 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr: 80.8% -- replaced with /var/log/ceph/cluster2-client.mirror.49490.log.gz 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr: 89.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78850.log.gz 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81243.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65917.log 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81606.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66313.log 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr: 89.8% -- replaced with /var/log/ceph/cluster2-client.mirror.81243.log.gz 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81606.log.gz 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65917.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64949.log 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.1.94153.log.gz 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.65917.log.gz 2026-02-22T22:10:37.910 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66313.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66603.log 2026-02-22T22:10:37.911 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.66313.log.gz 2026-02-22T22:10:37.911 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64949.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69590.log 2026-02-22T22:10:37.911 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.64949.log.gz 2026-02-22T22:10:37.911 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81875.log 2026-02-22T22:10:37.911 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.66603.log.gz 2026-02-22T22:10:37.911 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69590.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85372.log 2026-02-22T22:10:37.911 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.69590.log.gz 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81875.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57814.log 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.81875.log.gz 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.85372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81718.log 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.85372.log.gz 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.57814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.87358.log 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.57814.log.gz 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81718.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55238.log 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81718.log.gz 2026-02-22T22:10:37.912 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.87358.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.87358.log.gz 2026-02-22T22:10:37.913 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67186.log 2026-02-22T22:10:37.913 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55238.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72068.log 2026-02-22T22:10:37.913 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55238.log.gz 2026-02-22T22:10:37.913 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67186.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94860.log 2026-02-22T22:10:37.913 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.67186.log.gz 2026-02-22T22:10:37.913 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.72068.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64795.log 2026-02-22T22:10:37.913 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72068.log.gz 2026-02-22T22:10:37.914 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94860.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57969.log 2026-02-22T22:10:37.914 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.94860.log.gz 2026-02-22T22:10:37.914 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64795.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72775.log 2026-02-22T22:10:37.914 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.57969.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53939.log 2026-02-22T22:10:37.914 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.64795.log.gz 2026-02-22T22:10:37.914 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.57969.log.gz 2026-02-22T22:10:37.914 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72775.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78708.log 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.72775.log.gz 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.53939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52104.log 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53939.log.gz 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78708.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96429.log 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78708.log.gz 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68407.log 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.96429.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.52104.log.gz 2026-02-22T22:10:37.915 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67699.log 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.68407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47930.log 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.96429.log.gz 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.68407.log.gz 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67699.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.86668.log 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.67699.log.gz 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.47930.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47930.log.gz 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77738.log 2026-02-22T22:10:37.916 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.86668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77171.log 2026-02-22T22:10:37.917 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.86668.log.gz 2026-02-22T22:10:37.917 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75738.log 2026-02-22T22:10:37.917 INFO:teuthology.orchestra.run.trial104.stderr: 90.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77738.log.gz 2026-02-22T22:10:37.917 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77171.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68676.log 2026-02-22T22:10:37.917 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.77171.log.gz 2026-02-22T22:10:37.917 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54434.log 2026-02-22T22:10:37.918 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.75738.log.gz 2026-02-22T22:10:37.918 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68676.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90567.log 2026-02-22T22:10:37.918 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.68676.log.gz 2026-02-22T22:10:37.918 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.54434.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48932.log 2026-02-22T22:10:37.918 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.54434.log.gz 2026-02-22T22:10:37.918 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90567.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93898.log 2026-02-22T22:10:37.918 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.90567.log.gz 2026-02-22T22:10:37.919 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48932.log: 80.3% -- replaced with /var/log/ceph/cluster1-client.admin.48932.log.gz 2026-02-22T22:10:37.919 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85140.log 2026-02-22T22:10:37.919 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54660.log 2026-02-22T22:10:37.919 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85140.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.93898.log.gz 2026-02-22T22:10:37.919 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.68372.log 2026-02-22T22:10:37.919 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54660.log: 89.1%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.85140.log.gz 2026-02-22T22:10:37.919 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77773.log 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.68372.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48224.log 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54660.log.gz 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.68372.log.gz 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77773.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92137.log 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48224.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77773.log.gz 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48224.log.gz 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53213.log 2026-02-22T22:10:37.920 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92137.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81041.log 2026-02-22T22:10:37.921 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.92137.log.gz 2026-02-22T22:10:37.921 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85545.log 2026-02-22T22:10:37.921 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55350.log 2026-02-22T22:10:37.921 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.53213.log.gz 2026-02-22T22:10:37.921 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.96210.log 2026-02-22T22:10:37.921 INFO:teuthology.orchestra.run.trial104.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.85545.log.gz 2026-02-22T22:10:37.922 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55350.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62574.log 2026-02-22T22:10:37.922 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.55350.log.gz 2026-02-22T22:10:37.922 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.96210.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67130.log 2026-02-22T22:10:37.922 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.96210.log.gz 2026-02-22T22:10:37.922 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74095.log 2026-02-22T22:10:37.922 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.62574.log.gz 2026-02-22T22:10:37.922 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.67130.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87997.log 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74095.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75376.log 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.67130.log.gz 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.74095.log.gz 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87997.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.90172.log 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87997.log.gz 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75376.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49256.log 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr: 88.4% -- replaced with /var/log/ceph/cluster2-client.admin.75376.log.gz 2026-02-22T22:10:37.923 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.90172.log: 20.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.90172.log.gz 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46909.log 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49256.log: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49256.log.gz 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53464.log 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.46909.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46909.log.gz 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71270.log 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91833.log 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81041.log.gz 2026-02-22T22:10:37.924 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.53464.log.gz 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71270.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72571.log 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.71270.log.gz 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91833.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67071.log 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-client.mirror.1.50571.log.gz 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91833.log.gz 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55459.log 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.72571.log.gz 2026-02-22T22:10:37.925 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67071.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49171.log 2026-02-22T22:10:37.926 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.67071.log.gz 2026-02-22T22:10:37.926 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.55459.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77693.log 2026-02-22T22:10:37.926 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.55459.log.gz 2026-02-22T22:10:37.926 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.49171.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77885.log 2026-02-22T22:10:37.926 INFO:teuthology.orchestra.run.trial104.stderr: 80.2% -- replaced with /var/log/ceph/cluster2-client.admin.49171.log.gz 2026-02-22T22:10:37.926 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.77693.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86842.log 2026-02-22T22:10:37.926 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77885.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73745.log 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.86842.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.76794.log 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73745.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.77693.log.gz 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster1-client.mirror.86842.log.gz 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.54475.log 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.73745.log.gz 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54519.log 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.76794.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.77885.log.gz 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.0.54475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77540.log 2026-02-22T22:10:37.927 INFO:teuthology.orchestra.run.trial104.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.76794.log.gz 2026-02-22T22:10:37.928 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.54519.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55145.log 2026-02-22T22:10:37.928 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54519.log.gz 2026-02-22T22:10:37.928 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77540.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69894.log 2026-02-22T22:10:37.928 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77540.log.gz 2026-02-22T22:10:37.928 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55145.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72033.log 2026-02-22T22:10:37.928 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55145.log.gz 2026-02-22T22:10:37.928 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.49490.log 2026-02-22T22:10:37.929 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.69894.log.gz 2026-02-22T22:10:37.929 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.72033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75254.log 2026-02-22T22:10:37.929 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72033.log.gz 2026-02-22T22:10:37.929 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.49490.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51358.log 2026-02-22T22:10:37.929 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.75254.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93125.log 2026-02-22T22:10:37.929 INFO:teuthology.orchestra.run.trial104.stderr: 89.1%/var/log/ceph/cluster2-client.mirror.51358.log: -- replaced with /var/log/ceph/cluster2-client.mirror.75254.log.gz 2026-02-22T22:10:37.929 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72715.log 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51358.log.gz 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81963.log 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72715.log: 91.5%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.2.49490.log.gz 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.60414.log 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.72715.log.gz 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54133.log 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.81963.log.gz 2026-02-22T22:10:37.930 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.60414.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73052.log 2026-02-22T22:10:37.931 INFO:teuthology.orchestra.run.trial104.stderr: 92.5% -- replaced with /var/log/ceph/cluster2-client.mirror.93125.log.gz 2026-02-22T22:10:37.931 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.60414.log.gz 2026-02-22T22:10:37.931 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.54133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80567.log 2026-02-22T22:10:37.931 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73052.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65222.log 2026-02-22T22:10:37.931 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.73052.log.gz 2026-02-22T22:10:37.931 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80567.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53662.log 2026-02-22T22:10:37.931 INFO:teuthology.orchestra.run.trial104.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.80567.log.gz 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65222.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55180.log 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.65222.log.gz 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53662.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74002.log 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55180.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85008.log 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.53662.log.gz 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55180.log.gz 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74002.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.51191.log 2026-02-22T22:10:37.932 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.74002.log.gz 2026-02-22T22:10:37.933 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose/var/log/ceph/cluster2-client.mirror.85008.log: -- /var/log/ceph/cluster1-client.admin.89897.log 2026-02-22T22:10:37.933 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.51191.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.86132.log 2026-02-22T22:10:37.933 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.89897.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83686.log 2026-02-22T22:10:37.933 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85008.log.gz 2026-02-22T22:10:37.933 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.89897.log.gz 2026-02-22T22:10:37.933 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.86132.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77149.log 2026-02-22T22:10:37.934 INFO:teuthology.orchestra.run.trial104.stderr: 92.3%/var/log/ceph/cluster1-client.mirror.83686.log: -- replaced with /var/log/ceph/cluster1-client.mirror.51191.log.gz 2026-02-22T22:10:37.934 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78775.log 2026-02-22T22:10:37.934 INFO:teuthology.orchestra.run.trial104.stderr: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.83686.log.gz 2026-02-22T22:10:37.934 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77149.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90824.log 2026-02-22T22:10:37.934 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.77149.log.gz 2026-02-22T22:10:37.934 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78775.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.54133.log.gz 2026-02-22T22:10:37.934 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94895.log 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90824.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52611.log 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.78775.log.gz 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87951.log 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.94895.log.gz 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52611.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74440.log 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr: 91.3%/var/log/ceph/cluster1-client.mirror.87951.log: -- replaced with /var/log/ceph/cluster1-client.mirror.90824.log.gz 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.52611.log.gz 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.83663.log 2026-02-22T22:10:37.935 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.87951.log.gz 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74440.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60224.log 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.74440.log.gz 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.83663.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97391.log 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.83663.log.gz 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60224.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.94275.log 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.60224.log.gz 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97391.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97391.log.gz 2026-02-22T22:10:37.936 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89023.log 2026-02-22T22:10:37.937 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.3.94275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82247.log 2026-02-22T22:10:37.937 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89023.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94630.log 2026-02-22T22:10:37.937 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82247.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57365.log 2026-02-22T22:10:37.937 INFO:teuthology.orchestra.run.trial104.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82247.log.gz 2026-02-22T22:10:37.937 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94630.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50226.log 2026-02-22T22:10:37.937 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94630.log.gz 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57365.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68472.log 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89023.log.gz 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.50226.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70966.log 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.50226.log.gz 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68472.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49358.log 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.68472.log.gz 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70966.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.3.94275.log.gz 2026-02-22T22:10:37.938 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.86133.log 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.70966.log.gz 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49358.log: 54.2% -- replaced with /var/log/ceph/cluster2-client.mirror.49358.log.gz 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79820.log 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.86133.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.86133.log.gz 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68741.log 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79820.log: 93.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.86132.log.gz 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89601.log 2026-02-22T22:10:37.939 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79820.log.gz 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91384.log 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.68741.log.gz 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.89601.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.86123.log 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.89601.log.gz 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91384.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88613.log 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.91384.log.gz 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.86123.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.86123.log.gz 2026-02-22T22:10:37.940 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.54964.log 2026-02-22T22:10:37.941 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.88613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72312.log 2026-02-22T22:10:37.941 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.88613.log.gz 2026-02-22T22:10:37.941 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.54964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.94155.log 2026-02-22T22:10:37.941 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72312.log: gzip -5 87.0% --verbose -- -- replaced with /var/log/ceph/cluster1-client.admin.54964.log.gz /var/log/ceph/cluster1-client.mirror.68301.log 2026-02-22T22:10:37.941 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.941 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.72312.log.gz 2026-02-22T22:10:37.941 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.2.94155.log: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.57365.log.gz 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.94155.log.gz 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77848.log 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.68301.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74677.log 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.77848.log: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.68301.log.gz 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr: 93.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50528.log.gz 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48028.log 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74677.log: 90.8% -- replaced with /var/log/ceph/cluster1-client.admin.77848.log.gz 2026-02-22T22:10:37.942 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86227.log 2026-02-22T22:10:37.943 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.74677.log.gz 2026-02-22T22:10:37.943 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.48028.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48028.log.gz 2026-02-22T22:10:37.943 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90199.log 2026-02-22T22:10:37.943 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86227.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.54580.log 2026-02-22T22:10:37.943 INFO:teuthology.orchestra.run.trial104.stderr: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.86227.log.gz 2026-02-22T22:10:37.943 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90199.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50528.log 2026-02-22T22:10:37.943 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.2.54580.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68952.log 2026-02-22T22:10:37.944 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.50528.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82045.log 2026-02-22T22:10:37.944 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62125.log 2026-02-22T22:10:37.944 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68952.log: /var/log/ceph/cluster2-client.mirror.82045.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63042.log 2026-02-22T22:10:37.944 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.68952.log.gz 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82045.log.gz 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81524.log 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.62125.log.gz 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78805.log 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.63042.log.gz 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81524.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62522.log 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81524.log.gz 2026-02-22T22:10:37.945 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78805.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66782.log 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.78805.log.gz 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.62522.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78322.log 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80053.log 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.62522.log.gz 87.2% 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.66782.log.gz 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78322.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.76878.log 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80053.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62668.log 2026-02-22T22:10:37.946 INFO:teuthology.orchestra.run.trial104.stderr: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78322.log.gz 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.76878.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49017.log 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62668.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63747.log 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.76878.log.gz 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.49017.log: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.80053.log.gz 88.3% 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.62668.log.gz 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr: 3.5%gzip -- replaced with /var/log/ceph/cluster1-client.mirror.49017.log.gz 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80096.log 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.63747.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81651.log 2026-02-22T22:10:37.947 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.63747.log.gz 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80096.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70104.log 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81651.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97440.log 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% 89.6% -- replaced with /var/log/ceph/cluster1-client.mirror.81651.log.gz -- replaced with /var/log/ceph/cluster2-client.mirror.80096.log.gz 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70104.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.85282.log 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.70104.log.gz 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97440.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97440.log.gz 2026-02-22T22:10:37.948 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80730.log 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose/var/log/ceph/cluster1-client.mirror.85282.log: -- /var/log/ceph/cluster1-client.admin.68068.log 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.85282.log.gz 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80730.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92934.log 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68068.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89401.log 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80730.log.gz 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.68068.log.gz 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92934.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78536.log 2026-02-22T22:10:37.949 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.92934.log.gz 2026-02-22T22:10:37.950 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89401.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68439.log 2026-02-22T22:10:37.950 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78536.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87332.log 2026-02-22T22:10:37.950 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68439.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.78536.log.gz 2026-02-22T22:10:37.950 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81387.log 2026-02-22T22:10:37.950 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.68439.log.gz 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87332.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.89401.log.gz 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58335.log 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87332.log.gz 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81387.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.50529.log 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58335.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93303.log 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.58335.log.gz 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.50529.log: gzip -5 --verbose 19.0% -- -- replaced with /var/log/ceph/cluster1-client.mirror.1.50529.log.gz /var/log/ceph/cluster1-client.mirror.1.86175.log 2026-02-22T22:10:37.951 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.952 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93303.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.81387.log.gz 2026-02-22T22:10:37.952 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92242.log 2026-02-22T22:10:37.952 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93303.log.gz 2026-02-22T22:10:37.952 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.86175.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82083.log 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62639.log 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92242.log.gz 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82083.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.95762.log 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62639.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54580.log 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.62639.log.gz 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.95762.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78027.log 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82083.log.gz 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.54580.log: 91.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.86175.log.gz 2026-02-22T22:10:37.953 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89536.log 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54580.log.gz 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78027.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57869.log 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89536.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.mirror.78027.log.gz 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57717.log 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.89536.log.gz 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.57869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.91296.log 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.57717.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52794.log 2026-02-22T22:10:37.954 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.57717.log.gz 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.91296.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66811.log 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.91296.log.gz 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52794.log: 91.4% -- replaced with /var/log/ceph/cluster1-client.mirror.57869.log.gz 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49552.log 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.66811.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.52794.log.gz 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88936.log 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.66811.log.gz 2026-02-22T22:10:37.955 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49552.log: 84.3%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.49552.log.gz -5 2026-02-22T22:10:37.956 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.87928.log 2026-02-22T22:10:37.956 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.88936.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61917.log 2026-02-22T22:10:37.956 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63935.log 2026-02-22T22:10:37.956 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87928.log.gz 2026-02-22T22:10:37.956 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61917.log: 86.5% -- replaced with /var/log/ceph/cluster1-client.mirror.88936.log.gz 2026-02-22T22:10:37.956 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85901.log 2026-02-22T22:10:37.956 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.61917.log.gz 2026-02-22T22:10:37.957 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.63935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69135.log 2026-02-22T22:10:37.957 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.85901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64570.log 2026-02-22T22:10:37.957 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.63935.log.gz 2026-02-22T22:10:37.957 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.85901.log.gz 2026-02-22T22:10:37.957 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69135.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57573.log 2026-02-22T22:10:37.957 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.69135.log.gz 2026-02-22T22:10:37.957 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50099.log 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% 94.5% -- replaced with /var/log/ceph/cluster2-client.admin.64570.log.gz 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.2.54580.log.gz 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57573.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74617.log 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57573.log.gz 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50099.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66404.log 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50099.log.gz 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74617.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49273.log 2026-02-22T22:10:37.958 INFO:teuthology.orchestra.run.trial104.stderr: 88.0%/var/log/ceph/cluster2-client.admin.66404.log: -- replaced with /var/log/ceph/cluster1-client.admin.74617.log.gz 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66751.log 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.66404.log.gz 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49273.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48567.log 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.49273.log.gz 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.44196.log 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.48567.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.66751.log.gz 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48567.log.gz 2026-02-22T22:10:37.959 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80957.log 2026-02-22T22:10:37.960 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.44196.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.44196.log.gz 2026-02-22T22:10:37.960 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81782.log 2026-02-22T22:10:37.960 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87033.log 2026-02-22T22:10:37.960 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61299.log 2026-02-22T22:10:37.960 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87033.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53182.log 2026-02-22T22:10:37.960 INFO:teuthology.orchestra.run.trial104.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.81782.log.gz 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.87033.log.gz 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.61299.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49239.log 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80957.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.61299.log.gz 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53182.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71208.log 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.80957.log.gz 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.53182.log.gz 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49239.log: gzip 2.3% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.49239.log.gz --verbose 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr: -- /var/log/ceph/cluster1-client.mirror.78511.log 2026-02-22T22:10:37.961 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.71208.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81132.log 2026-02-22T22:10:37.962 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.71208.log.gz 2026-02-22T22:10:37.962 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78511.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67939.log 2026-02-22T22:10:37.962 INFO:teuthology.orchestra.run.trial104.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.78511.log.gz 2026-02-22T22:10:37.962 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58908.log 2026-02-22T22:10:37.962 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.67939.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.81132.log.gz 2026-02-22T22:10:37.962 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75498.log 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67939.log.gz 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58908.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52073.log 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.58908.log.gz 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75498.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78591.log 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52073.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.75498.log.gz 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90243.log 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.52073.log.gz 2026-02-22T22:10:37.963 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78591.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51101.log 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.78591.log.gz 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90243.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92265.log 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr: 86.6% -- replaced with /var/log/ceph/cluster2-client.mirror.90243.log.gz 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.51101.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.54993.log 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.51101.log.gz 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92265.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65738.log 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.92265.log.gz 2026-02-22T22:10:37.964 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.54993.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68213.log 2026-02-22T22:10:37.965 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.54993.log.gz 2026-02-22T22:10:37.965 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65738.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.72400.log 2026-02-22T22:10:37.965 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.65738.log.gz 2026-02-22T22:10:37.965 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68213.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83310.log 2026-02-22T22:10:37.965 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.68213.log.gz 2026-02-22T22:10:37.965 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.72400.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79205.log 2026-02-22T22:10:37.965 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.72400.log.gz 2026-02-22T22:10:37.966 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.83310.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66691.log 2026-02-22T22:10:37.966 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.83310.log.gz 2026-02-22T22:10:37.966 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79205.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.95122.log 2026-02-22T22:10:37.966 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.66691.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63907.log 2026-02-22T22:10:37.966 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79205.log.gz 2026-02-22T22:10:37.966 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.95122.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53582.log 2026-02-22T22:10:37.966 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.mirror.66691.log.gz 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.95122.log.gz 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63907.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93052.log 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53582.log: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.63907.log.gz 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51865.log 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93052.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53582.log.gz 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66538.log 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93052.log.gz 2026-02-22T22:10:37.967 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77607.log 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.66538.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51865.log.gz 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89983.log 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.66538.log.gz 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.77607.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94655.log 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89983.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.77607.log.gz 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.64144.log 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94655.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52269.log 2026-02-22T22:10:37.968 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94655.log.gz 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.64144.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.71389.log 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52269.log: 91.3% -- replaced with /var/log/ceph/cluster1-client.mirror.89983.log.gz 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- 89.1% /var/log/ceph/cluster1-client.admin.71843.log 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.64144.log.gz 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66663.log 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.71389.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.52269.log.gz 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.71843.log: gzip -5 88.2% --verbose -- replaced with /var/log/ceph/cluster1-client.admin.71389.log.gz -- 2026-02-22T22:10:37.969 INFO:teuthology.orchestra.run.trial104.stderr: /var/log/ceph/cluster2-client.admin.59605.log 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.66663.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.71843.log.gz 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66570.log 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.66663.log.gz 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.59605.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97537.log 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.59605.log.gz 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66570.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63876.log 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.66570.log.gz 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78685.log 2026-02-22T22:10:37.970 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97537.log.gz 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93269.log 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.63876.log.gz 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78685.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50134.log 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78685.log.gz 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93269.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81453.log 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.93269.log.gz 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50134.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62463.log 2026-02-22T22:10:37.971 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50134.log.gz 2026-02-22T22:10:37.972 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81453.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50356.log 2026-02-22T22:10:37.972 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81453.log.gz 2026-02-22T22:10:37.972 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70319.log 2026-02-22T22:10:37.972 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.62463.log.gz 2026-02-22T22:10:37.972 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.50356.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89491.log 2026-02-22T22:10:37.972 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50356.log.gz 2026-02-22T22:10:37.972 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70319.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67246.log 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.70319.log.gz 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89491.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53005.log 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.89491.log.gz 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67246.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66632.log 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr: 95.3% -- replaced with /var/log/ceph/cluster1-client.mirror.2.95762.log.gz 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.67246.log.gz 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53005.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68273.log 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.53005.log.gz 2026-02-22T22:10:37.973 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.66632.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46999.log 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68273.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.66632.log.gz 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97244.log 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.68273.log.gz 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.46999.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61950.log 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46999.log.gz 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57186.log 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97244.log.gz 2026-02-22T22:10:37.974 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61950.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.49552.log 2026-02-22T22:10:37.975 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.61950.log.gz 2026-02-22T22:10:37.975 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.57186.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89756.log 2026-02-22T22:10:37.975 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.57186.log.gz 2026-02-22T22:10:37.975 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.49552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88244.log 2026-02-22T22:10:37.975 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89756.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58363.log 2026-02-22T22:10:37.975 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.89756.log.gz 2026-02-22T22:10:37.975 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88244.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78966.log 2026-02-22T22:10:37.976 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.58363.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64331.log 2026-02-22T22:10:37.976 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78966.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54657.log 2026-02-22T22:10:37.976 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.58363.log.gz 2026-02-22T22:10:37.976 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64331.log: 91.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.49552.log.gz 2026-02-22T22:10:37.976 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62958.log 2026-02-22T22:10:37.976 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.64331.log.gz 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.54657.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77575.log 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.54657.log.gz 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62958.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47587.log 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr: 90.5% -- replaced with /var/log/ceph/cluster1-client.mirror.78966.log.gz 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.62958.log.gz 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77575.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55492.log 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77575.log.gz 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47587.log: gzip 0.0% -5 -- replaced with /var/log/ceph/cluster2-client.admin.47587.log.gz --verbose 2026-02-22T22:10:37.977 INFO:teuthology.orchestra.run.trial104.stderr: -- /var/log/ceph/cluster2-client.mirror.77130.log 2026-02-22T22:10:37.978 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.55492.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60011.log 2026-02-22T22:10:37.978 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.55492.log.gz 2026-02-22T22:10:37.978 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77130.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65705.log 2026-02-22T22:10:37.978 INFO:teuthology.orchestra.run.trial104.stderr: 83.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77130.log.gz 2026-02-22T22:10:37.978 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.60011.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80624.log 2026-02-22T22:10:37.978 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.60011.log.gz 2026-02-22T22:10:37.978 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65705.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65255.log 2026-02-22T22:10:37.979 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.65705.log.gz 2026-02-22T22:10:37.979 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80624.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82752.log 2026-02-22T22:10:37.979 INFO:teuthology.orchestra.run.trial104.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.80624.log.gz 2026-02-22T22:10:37.979 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65255.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73596.log 2026-02-22T22:10:37.979 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.65255.log.gz 2026-02-22T22:10:37.979 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.88244.log.gz 2026-02-22T22:10:37.979 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82752.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68242.log 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr: 63.6% -- replaced with /var/log/ceph/cluster2-client.mirror.82752.log.gz 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.68887.log 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.73596.log.gz 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68242.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51533.log 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.68887.log: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.68242.log.gz 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76466.log 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.68887.log.gz 2026-02-22T22:10:37.980 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53036.log 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.76466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87471.log 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51533.log.gz 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.76466.log.gz 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53036.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.66432.log 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87471.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.53036.log.gz 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70833.log 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr: 84.9% -- replaced with /var/log/ceph/cluster2-client.mirror.87471.log.gz 2026-02-22T22:10:37.981 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.66432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80004.log 2026-02-22T22:10:37.982 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70833.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.66432.log.gz 2026-02-22T22:10:37.982 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55427.log 2026-02-22T22:10:37.982 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.admin.70833.log.gz 2026-02-22T22:10:37.982 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80004.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82867.log 2026-02-22T22:10:37.982 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55427.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.85934.log 2026-02-22T22:10:37.982 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82867.log: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.55427.log.gz 2026-02-22T22:10:37.982 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66130.log 2026-02-22T22:10:37.983 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.80004.log.gz 2026-02-22T22:10:37.983 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.85934.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72133.log 2026-02-22T22:10:37.983 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.66130.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.85934.log.gz 2026-02-22T22:10:37.983 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67102.log 2026-02-22T22:10:37.983 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.66130.log.gz 2026-02-22T22:10:37.983 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72133.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93485.log 2026-02-22T22:10:37.983 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.72133.log.gz 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74373.log 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.67102.log.gz 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93485.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63313.log 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.admin.93485.log.gz 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.74373.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78996.log 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74373.log.gz 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.63313.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95172.log 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63313.log.gz 2026-02-22T22:10:37.984 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78996.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.74922.log 2026-02-22T22:10:37.985 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.78996.log.gz 2026-02-22T22:10:37.985 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.95172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75195.log 2026-02-22T22:10:37.985 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.74922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91650.log 2026-02-22T22:10:37.985 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.95172.log.gz 2026-02-22T22:10:37.985 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.74922.log.gz 2026-02-22T22:10:37.985 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75195.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58660.log 2026-02-22T22:10:37.985 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.75195.log.gz 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91650.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79743.log 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91650.log.gz 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.58660.log: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.82867.log.gz 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95150.log 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.58660.log.gz 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79743.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.50572.log 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79743.log.gz 2026-02-22T22:10:37.986 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.95150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77673.log 2026-02-22T22:10:37.987 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.50572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.75309.log 2026-02-22T22:10:37.987 INFO:teuthology.orchestra.run.trial104.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.2.50572.log.gz 2026-02-22T22:10:37.987 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47517.log 2026-02-22T22:10:37.987 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.75309.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.50519.log 2026-02-22T22:10:37.987 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.75309.log.gz 2026-02-22T22:10:37.987 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47517.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.79887.log 2026-02-22T22:10:37.987 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47517.log.gz 2026-02-22T22:10:37.988 INFO:teuthology.orchestra.run.trial104.stderr: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77673.log.gz 2026-02-22T22:10:37.988 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.50519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.77817.log 2026-02-22T22:10:37.988 INFO:teuthology.orchestra.run.trial104.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50519.log.gz 2026-02-22T22:10:37.988 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.79887.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72162.log 2026-02-22T22:10:37.988 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.79887.log.gz 2026-02-22T22:10:37.988 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.77817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49051.log 2026-02-22T22:10:37.988 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72162.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.77817.log.gz 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93537.log 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.72162.log.gz 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.49051.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72193.log 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr: 57.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49051.log.gz 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93537.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89731.log 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.93537.log.gz 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72193.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80243.log 2026-02-22T22:10:37.989 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.72193.log.gz 2026-02-22T22:10:37.990 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89731.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52880.log 2026-02-22T22:10:37.990 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.89731.log.gz 2026-02-22T22:10:37.990 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80243.log: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.95150.log.gz 2026-02-22T22:10:37.990 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77928.log 2026-02-22T22:10:37.990 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52880.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50336.log 2026-02-22T22:10:37.990 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77928.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.52880.log.gz 2026-02-22T22:10:37.990 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88334.log 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.50336.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77928.log.gz 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.49432.log 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.mirror.50336.log.gz 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88334.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69016.log 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88334.log.gz 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.49432.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70995.log 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.49432.log.gz 2026-02-22T22:10:37.991 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.63378.log 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.69016.log.gz 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70995.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81806.log 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.70995.log.gz 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.63378.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59422.log 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.63378.log.gz 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81806.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.76176.log 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81806.log.gz 2026-02-22T22:10:37.992 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.59422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62094.log 2026-02-22T22:10:37.993 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.59422.log.gz 2026-02-22T22:10:37.993 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.76176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81431.log 2026-02-22T22:10:37.993 INFO:teuthology.orchestra.run.trial104.stderr: 86.3% -- replaced with /var/log/ceph/cluster2-client.admin.76176.log.gz 2026-02-22T22:10:37.993 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62094.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66941.log 2026-02-22T22:10:37.993 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.62094.log.gz 2026-02-22T22:10:37.993 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.80243.log.gz 2026-02-22T22:10:37.993 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96950.log 2026-02-22T22:10:37.994 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.66941.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.58542.log 2026-02-22T22:10:37.994 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81431.log.gz 2026-02-22T22:10:37.994 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66941.log.gz 2026-02-22T22:10:37.994 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.96950.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96950.log.gz 2026-02-22T22:10:37.994 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79608.log 2026-02-22T22:10:37.994 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.58542.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70044.log 2026-02-22T22:10:37.994 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79608.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.58542.log.gz 2026-02-22T22:10:37.995 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88546.log 2026-02-22T22:10:37.995 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79608.log.gz 2026-02-22T22:10:37.995 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.96326.log 2026-02-22T22:10:37.995 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.70044.log.gz 2026-02-22T22:10:37.995 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88546.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67277.log 2026-02-22T22:10:37.995 INFO:teuthology.orchestra.run.trial104.stderr: 86.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88546.log.gz 2026-02-22T22:10:37.995 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.96326.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71783.log 2026-02-22T22:10:37.996 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67277.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60749.log 2026-02-22T22:10:37.996 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.67277.log.gz 2026-02-22T22:10:37.996 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.71783.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69104.log 2026-02-22T22:10:37.996 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60749.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.71783.log.gz 2026-02-22T22:10:37.996 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73293.log 2026-02-22T22:10:37.996 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.60749.log.gz 2026-02-22T22:10:37.996 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69104.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61449.log 2026-02-22T22:10:37.997 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.69104.log.gz 2026-02-22T22:10:37.997 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.69499.log 2026-02-22T22:10:37.997 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.73293.log.gz 2026-02-22T22:10:37.997 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61449.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.91327.log 2026-02-22T22:10:37.997 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.61449.log.gz 2026-02-22T22:10:37.997 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.69499.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58238.log 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.91327.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68616.log 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.69499.log.gz 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.91327.log.gz 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58238.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50166.log 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.58238.log.gz 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68616.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.57781.log 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.50166.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.68616.log.gz 2026-02-22T22:10:37.998 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.71508.log 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.50166.log.gz 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.57781.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56889.log 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.96326.log.gz 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.71508.log: gzip -5 88.5% --verbose -- -- replaced with /var/log/ceph/cluster2-client.admin.57781.log.gz /var/log/ceph/cluster2-client.mirror.84810.log 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56889.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.71508.log.gz 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53831.log 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56889.log.gz 2026-02-22T22:10:37.999 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84810.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63557.log 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53831.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52042.log 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84810.log.gz 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65769.log 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.63557.log.gz 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52042.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.87410.log 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65769.log: 90.3% -- replaced with /var/log/ceph/cluster1-client.admin.52042.log.gz 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.82149.log 2026-02-22T22:10:38.000 INFO:teuthology.orchestra.run.trial104.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.53831.log.gz 2026-02-22T22:10:38.001 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.65769.log.gz 2026-02-22T22:10:38.001 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.87410.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75016.log 2026-02-22T22:10:38.001 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.82149.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81064.log 2026-02-22T22:10:38.001 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.82149.log.gz 2026-02-22T22:10:38.001 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75016.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73407.log 2026-02-22T22:10:38.001 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.75016.log.gz 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81064.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89800.log 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81064.log.gz 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.73407.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51896.log 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73407.log.gz 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89800.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89003.log 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr: 92.6% -- replaced with /var/log/ceph/cluster1-client.mirror.1.87410.log.gz 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.89800.log.gz 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51896.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.95660.log 2026-02-22T22:10:38.002 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89003.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.51896.log.gz 2026-02-22T22:10:38.003 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56081.log 2026-02-22T22:10:38.003 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.89003.log.gz 2026-02-22T22:10:38.003 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.1.95660.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.67612.log 2026-02-22T22:10:38.003 INFO:teuthology.orchestra.run.trial104.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.1.95660.log.gz 2026-02-22T22:10:38.003 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.56081.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.82734.log 2026-02-22T22:10:38.003 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.56081.log.gz 2026-02-22T22:10:38.003 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.67612.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86132.log 2026-02-22T22:10:38.004 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.82734.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88808.log 2026-02-22T22:10:38.004 INFO:teuthology.orchestra.run.trial104.stderr: 80.0% 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.82734.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.67612.log.gz 2026-02-22T22:10:38.004 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:38.004 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75904.log 2026-02-22T22:10:38.004 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.60782.log 2026-02-22T22:10:38.004 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.88808.log: /var/log/ceph/cluster1-client.admin.75904.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65284.log 2026-02-22T22:10:38.004 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.75904.log.gz 2026-02-22T22:10:38.005 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.60782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66094.log 2026-02-22T22:10:38.005 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.60782.log.gz 2026-02-22T22:10:38.005 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65284.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48420.log 2026-02-22T22:10:38.005 INFO:teuthology.orchestra.run.trial104.stderr: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.88808.log.gz 2026-02-22T22:10:38.005 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.66094.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.65284.log.gz 2026-02-22T22:10:38.005 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62010.log 2026-02-22T22:10:38.005 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.66094.log.gz 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.48420.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48420.log.gz 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60963.log 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr: 94.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86132.log.gz 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93411.log 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62010.log: /var/log/ceph/cluster1-client.mirror.60963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95658.log 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.60963.log.gz 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.62010.log.gz 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93411.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89667.log 2026-02-22T22:10:38.006 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.93411.log.gz 2026-02-22T22:10:38.007 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.95658.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81551.log 2026-02-22T22:10:38.007 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster2-client.mirror.95658.log.gz 2026-02-22T22:10:38.007 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80667.log 2026-02-22T22:10:38.007 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81551.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58696.log 2026-02-22T22:10:38.007 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81551.log.gz 2026-02-22T22:10:38.007 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.86971.log 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.89667.log.gz 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58696.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59817.log 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80667.log.gz 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster2-client.admin.58696.log.gz 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.86971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63470.log 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.86971.log.gz 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59817.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.63278.log 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.59817.log.gz 2026-02-22T22:10:38.008 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.63470.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80688.log 2026-02-22T22:10:38.009 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.63278.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83633.log 2026-02-22T22:10:38.009 INFO:teuthology.orchestra.run.trial104.stderr: 89.9% -- replaced with /var/log/ceph/cluster2-client.mirror.63470.log.gz 2026-02-22T22:10:38.009 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.63278.log.gz 2026-02-22T22:10:38.009 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80688.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59150.log 2026-02-22T22:10:38.009 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.83633.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81741.log 2026-02-22T22:10:38.009 INFO:teuthology.orchestra.run.trial104.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.80688.log.gz 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59150.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69619.log 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.59150.log.gz 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81741.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62986.log 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81741.log.gz 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.69619.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.mirror.83633.log.gz 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61028.log 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.69619.log.gz 2026-02-22T22:10:38.010 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.62986.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81329.log 2026-02-22T22:10:38.011 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70901.log 2026-02-22T22:10:38.011 INFO:teuthology.orchestra.run.trial104.stderr: 89.8% -- replaced with /var/log/ceph/cluster1-client.mirror.62986.log.gz 2026-02-22T22:10:38.011 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.61028.log.gz 2026-02-22T22:10:38.011 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81329.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55719.log 2026-02-22T22:10:38.011 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70901.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51265.log 2026-02-22T22:10:38.011 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.81329.log.gz 2026-02-22T22:10:38.011 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.70901.log.gz 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.55719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.65009.log 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.55719.log.gz 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.51265.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80299.log 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51265.log.gz 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.65009.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91685.log 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.65009.log.gz 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80299.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.61779.log 2026-02-22T22:10:38.012 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.80299.log.gz 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91685.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.60928.log 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91685.log.gz 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.61779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61979.log 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr: 89.0% -- replaced with /var/log/ceph/cluster1-client.mirror.61779.log.gz 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.60928.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.54475.log 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.60928.log.gz 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.65151.log 2026-02-22T22:10:38.013 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.61979.log.gz 2026-02-22T22:10:38.014 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.54475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84788.log 2026-02-22T22:10:38.014 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.65151.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77482.log 2026-02-22T22:10:38.014 INFO:teuthology.orchestra.run.trial104.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.65151.log.gz 2026-02-22T22:10:38.014 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84788.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58729.log 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.3.54657.log 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.84788.log.gz 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.77482.log.gz 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58729.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83544.log 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.58729.log.gz 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.3.54657.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.50700.log 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.83544.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.83544.log.gz 2026-02-22T22:10:38.015 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71301.log 2026-02-22T22:10:38.016 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.50700.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43954.log 2026-02-22T22:10:38.016 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71301.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.66842.log 2026-02-22T22:10:38.016 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.71301.log.gz 2026-02-22T22:10:38.016 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.43954.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43954.log.gz 2026-02-22T22:10:38.016 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94290.log 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75436.log 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.66842.log: /var/log/ceph/cluster1-client.mirror.94290.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62430.log 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.94290.log.gz 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.66842.log.gz 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75436.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50571.log 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62430.log: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.75436.log.gz 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.1.49390.log 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.62430.log.gz 2026-02-22T22:10:38.017 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93960.log 2026-02-22T22:10:38.018 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50571.log: /var/log/ceph/cluster1-client.mirror.1.49390.log: gzip -5 --verbose 19.6% -- -- replaced with /var/log/ceph/cluster1-client.mirror.1.49390.log.gz /var/log/ceph/cluster1-client.admin.75162.log 2026-02-22T22:10:38.018 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:38.018 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93960.log: 86.8%gzip -- replaced with /var/log/ceph/cluster1-client.admin.93960.log.gz -5 2026-02-22T22:10:38.018 INFO:teuthology.orchestra.run.trial104.stderr: --verbose -- /var/log/ceph/cluster1-client.admin.72806.log 2026-02-22T22:10:38.018 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89047.log 2026-02-22T22:10:38.019 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75162.log: /var/log/ceph/cluster1-client.admin.72806.log: gzip -5 --verbose 87.2% -- -- replaced with /var/log/ceph/cluster1-client.admin.75162.log.gz /var/log/ceph/cluster1-client.mirror.92109.log 2026-02-22T22:10:38.019 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:38.019 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.72806.log.gz 2026-02-22T22:10:38.019 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89047.log: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.89047.log.gz 2026-02-22T22:10:38.019 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70215.log 2026-02-22T22:10:38.019 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92109.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97146.log 2026-02-22T22:10:38.019 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.70215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.55607.log 2026-02-22T22:10:38.020 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.92109.log.gz 2026-02-22T22:10:38.020 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.70215.log.gz 2026-02-22T22:10:38.020 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97146.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97146.log.gz 2026-02-22T22:10:38.020 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.81308.log 2026-02-22T22:10:38.020 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93726.log 2026-02-22T22:10:38.020 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.81308.log: /var/log/ceph/cluster1-client.mirror.55607.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73114.log 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.81308.log.gz 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.93726.log: 92.2% -- replaced with /var/log/ceph/cluster2-client.mirror.50571.log.gz 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.90302.log 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73114.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.53864.log 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.93726.log.gz 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.90302.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.73114.log.gz 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.55607.log.gz 2026-02-22T22:10:38.021 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47198.log 2026-02-22T22:10:38.022 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.53864.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.mirror.53864.log.gz 2026-02-22T22:10:38.022 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.94018.log 2026-02-22T22:10:38.022 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47198.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.95765.log 2026-02-22T22:10:38.022 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47198.log.gz 2026-02-22T22:10:38.022 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.94018.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53067.log 2026-02-22T22:10:38.022 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.94018.log.gz 2026-02-22T22:10:38.023 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.95765.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62330.log 2026-02-22T22:10:38.023 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.3.95765.log.gz 2026-02-22T22:10:38.023 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92843.log 2026-02-22T22:10:38.023 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53067.log: /var/log/ceph/cluster2-client.mirror.62330.log: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.3.90302.log.gz 2026-02-22T22:10:38.023 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.53067.log.gz 2026-02-22T22:10:38.023 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88981.log 2026-02-22T22:10:38.023 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62330.log.gz 2026-02-22T22:10:38.024 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92843.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79507.log 2026-02-22T22:10:38.024 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.92843.log.gz 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.92715.log/var/log/ceph/cluster1-client.mirror.88981.log: 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.88981.log.gz 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79507.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58968.log 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.mirror.79507.log.gz 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56924.log 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.92715.log: /var/log/ceph/cluster2-client.admin.58968.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80024.log 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.92715.log.gz 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.58968.log.gz 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56924.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78655.log 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.56924.log.gz 2026-02-22T22:10:38.025 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48518.log 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.54475.log.gz 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78557.log 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78655.log: /var/log/ceph/cluster2-client.admin.48518.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48518.log.gz 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56843.log 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.80024.log.gz 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78557.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88673.log 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.mirror.78557.log.gz 2026-02-22T22:10:38.026 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56843.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90804.log 2026-02-22T22:10:38.027 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.56843.log.gz 2026-02-22T22:10:38.027 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.88673.log: 90.9% -- replaced with /var/log/ceph/cluster2-client.mirror.78655.log.gz 2026-02-22T22:10:38.027 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70802.log 2026-02-22T22:10:38.027 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.88673.log.gz 2026-02-22T22:10:38.027 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90804.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.59271.log 2026-02-22T22:10:38.027 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70802.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.62699.log 2026-02-22T22:10:38.027 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.70802.log.gz 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.59271.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81696.log 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-client.mirror.90804.log.gz 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.62699.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.59271.log.gz 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.75047.log 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.62699.log.gz 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81696.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91720.log 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.81696.log.gz 2026-02-22T22:10:38.028 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.75047.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52353.log 2026-02-22T22:10:38.029 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.75047.log.gz 2026-02-22T22:10:38.029 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91720.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.72985.log 2026-02-22T22:10:38.029 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91720.log.gz 2026-02-22T22:10:38.029 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52353.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82593.log 2026-02-22T22:10:38.029 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.52353.log.gz 2026-02-22T22:10:38.029 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.72985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97342.log 2026-02-22T22:10:38.029 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.72985.log.gz 2026-02-22T22:10:38.030 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82593.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49188.log 2026-02-22T22:10:38.030 INFO:teuthology.orchestra.run.trial104.stderr: 94.6% -- replaced with /var/log/ceph/cluster2-client.mirror.3.54657.log.gz 2026-02-22T22:10:38.030 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97342.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97342.log.gz 2026-02-22T22:10:38.030 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.53523.log 2026-02-22T22:10:38.030 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.49188.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74274.log 2026-02-22T22:10:38.030 INFO:teuthology.orchestra.run.trial104.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.49188.log.gz 2026-02-22T22:10:38.030 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.53523.log: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.82593.log.gz 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52415.log 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.53523.log.gz 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74274.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83275.log 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.74274.log.gz 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52415.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81179.log 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.52415.log.gz 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.83275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52176.log 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.83275.log.gz 2026-02-22T22:10:38.031 INFO:teuthology.orchestra.run.trial104.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.54475.log.gz 2026-02-22T22:10:38.032 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81179.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69044.log 2026-02-22T22:10:38.032 INFO:teuthology.orchestra.run.trial104.stderr: 94.5% -- replaced with /var/log/ceph/cluster1-client.mirror.3.50700.log.gz 2026-02-22T22:10:38.032 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.81179.log.gz 2026-02-22T22:10:38.032 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.91113.log 2026-02-22T22:10:38.032 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.52176.log.gz 2026-02-22T22:10:38.032 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.69044.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.66870.log 2026-02-22T22:10:38.032 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.91113.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.71137.log 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.69044.log.gz 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.91113.log.gz 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.66870.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.95649.log 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.66870.log.gz 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.90110.log 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.71137.log: /var/log/ceph/cluster1-client.mirror.0.95649.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.55779.log 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.95649.log.gz 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.71137.log.gz 2026-02-22T22:10:38.033 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.90110.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.61382.log 2026-02-22T22:10:38.034 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.90110.log.gz 2026-02-22T22:10:38.034 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.55779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.54466.log 2026-02-22T22:10:38.034 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.admin.55779.log.gz 2026-02-22T22:10:38.034 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.61382.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64730.log 2026-02-22T22:10:38.034 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.61382.log.gz 2026-02-22T22:10:38.034 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.0.54466.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56215.log 2026-02-22T22:10:38.034 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.54466.log.gz 2026-02-22T22:10:38.035 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.64730.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.68004.log 2026-02-22T22:10:38.035 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.64730.log.gz 2026-02-22T22:10:38.035 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.56215.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62727.log 2026-02-22T22:10:38.035 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.56215.log.gz 2026-02-22T22:10:38.035 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.68004.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93871.log 2026-02-22T22:10:38.035 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.68004.log.gz 2026-02-22T22:10:38.035 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.62727.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73145.log 2026-02-22T22:10:38.036 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.62727.log.gz 2026-02-22T22:10:38.036 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.93871.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.73776.log 2026-02-22T22:10:38.036 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.mirror.93871.log.gz 2026-02-22T22:10:38.036 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73145.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74502.log 2026-02-22T22:10:38.036 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.73145.log.gz 2026-02-22T22:10:38.036 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.73776.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58275.log 2026-02-22T22:10:38.036 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.73776.log.gz 2026-02-22T22:10:38.037 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81673.log 2026-02-22T22:10:38.037 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.74502.log.gz 2026-02-22T22:10:38.037 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58275.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.57104.log 2026-02-22T22:10:38.037 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.58275.log.gz 2026-02-22T22:10:38.037 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59450.log 2026-02-22T22:10:38.037 INFO:teuthology.orchestra.run.trial104.stderr: 86.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81673.log.gz 2026-02-22T22:10:38.037 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.57104.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47621.log 2026-02-22T22:10:38.038 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.59450.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64510.log 2026-02-22T22:10:38.038 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47621.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59450.log.gz 2026-02-22T22:10:38.038 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47621.log.gz 2026-02-22T22:10:38.038 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.76204.log 2026-02-22T22:10:38.038 INFO:teuthology.orchestra.run.trial104.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-client.mirror.57104.log.gz 2026-02-22T22:10:38.038 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.58789.log 2026-02-22T22:10:38.038 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64510.log: /var/log/ceph/cluster2-client.mirror.76204.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86512.log 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.64510.log.gz 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.58789.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.76204.log.gz 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96370.log 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.58789.log.gz 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.86512.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.64598.log 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.86512.log.gz 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.96370.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92196.log 2026-02-22T22:10:38.039 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.64598.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47881.log 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92196.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.64598.log.gz 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.59877.log 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr: 86.7% -- replaced with /var/log/ceph/cluster1-client.mirror.92196.log.gz 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.47881.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47881.log.gz 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79248.log 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.59877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.97293.log 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.59877.log.gz 2026-02-22T22:10:38.040 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79248.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90302.log 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.97293.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52384.log 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.97293.log.gz 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.79248.log.gz 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90302.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.62798.log 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.mirror.90302.log.gz 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52384.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.69374.log 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.62798.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52384.log.gz 2026-02-22T22:10:38.041 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47200.log 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.62798.log.gz 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.69374.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.87412.log 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.47200.log: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.69374.log.gz 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47200.log.gz 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88644.log 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr: 96.1% -- replaced with /var/log/ceph/cluster1-client.mirror.96370.log.gz 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.87412.log: gzip -5 --verbose 19.6% -- /var/log/ceph/cluster2-client.mirror.79855.log -- replaced with /var/log/ceph/cluster1-client.mirror.2.87412.log.gz 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:38.042 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.88644.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78894.log 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.88644.log.gz 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79855.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52580.log 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.79855.log.gz 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78894.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59667.log 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.78894.log.gz 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52580.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.95822.log 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.52580.log.gz 2026-02-22T22:10:38.043 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59667.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88854.log 2026-02-22T22:10:38.044 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.59667.log.gz 2026-02-22T22:10:38.044 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.95822.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77079.log 2026-02-22T22:10:38.044 INFO:teuthology.orchestra.run.trial104.stderr: 88.7% -- replaced with /var/log/ceph/cluster2-client.mirror.95822.log.gz 2026-02-22T22:10:38.044 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88854.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93238.log 2026-02-22T22:10:38.044 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77079.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.69953.log 2026-02-22T22:10:38.044 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.mirror.77079.log.gz 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93238.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62365.log 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.mirror.93238.log.gz 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.69953.log: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88854.log.gz 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93017.log 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr:gzip 89.8% -5 -- replaced with /var/log/ceph/cluster2-client.mirror.69953.log.gz --verbose 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr: -- /var/log/ceph/cluster1-client.mirror.59905.log 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93017.log: /var/log/ceph/cluster2-client.mirror.62365.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.96999.log 2026-02-22T22:10:38.045 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.93017.log.gz 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.62365.log.gz 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.59905.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58206.log 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.59905.log.gz 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.96999.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.96999.log.gz 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.81497.log 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.58206.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53811.log 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58206.log.gz 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.81497.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55587.log 2026-02-22T22:10:38.046 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.81497.log.gz 2026-02-22T22:10:38.047 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.89830.log 2026-02-22T22:10:38.047 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.53811.log.gz 2026-02-22T22:10:38.047 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55587.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87535.log 2026-02-22T22:10:38.047 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.89830.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93431.log 2026-02-22T22:10:38.047 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.89830.log.gz 2026-02-22T22:10:38.047 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87535.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.82638.log 2026-02-22T22:10:38.047 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87535.log.gz 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster2-client.mirror.55587.log.gz 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93431.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73836.log 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.82638.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.82638.log.gz 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63588.log 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73836.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80126.log 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.73836.log.gz 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63588.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.62247.log 2026-02-22T22:10:38.048 INFO:teuthology.orchestra.run.trial104.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.93431.log.gz 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.63588.log.gz 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80126.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.82478.log 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.62247.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57608.log 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.82478.log: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.62247.log.gz 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.74708.log 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.82478.log.gz 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57608.log: gzip -5 90.7% --verbose -- -- replaced with /var/log/ceph/cluster2-client.mirror.80126.log.gz /var/log/ceph/cluster1-client.admin.51502.log 2026-02-22T22:10:38.049 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.57608.log.gz 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.74708.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48371.log 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.74708.log.gz 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51502.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.55552.log 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.51502.log.gz 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48371.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48371.log.gz 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79955.log 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.55552.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94067.log 2026-02-22T22:10:38.050 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.55552.log.gz 2026-02-22T22:10:38.051 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79955.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.59726.log 2026-02-22T22:10:38.051 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94067.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.78156.log 2026-02-22T22:10:38.051 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.59726.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67584.log 2026-02-22T22:10:38.051 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.78156.log: 93.1% -- replaced with /var/log/ceph/cluster2-client.mirror.79955.log.gz 2026-02-22T22:10:38.051 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48983.log 2026-02-22T22:10:38.051 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.59726.log.gz 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67584.log: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.78156.log.gz 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr: 90.8% -- replaced with /var/log/ceph/cluster2-client.mirror.94067.log.gz 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57895.log 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.67584.log.gz 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.48983.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72100.log 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48983.log.gz 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57895.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55052.log 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72100.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.71449.log 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.57895.log.gz 2026-02-22T22:10:38.052 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.72100.log.gz 2026-02-22T22:10:38.053 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48273.log 2026-02-22T22:10:38.053 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.71449.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.55052.log.gz 2026-02-22T22:10:38.053 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.65640.log 2026-02-22T22:10:38.053 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.71449.log.gz 2026-02-22T22:10:38.053 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48273.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48273.log.gz 2026-02-22T22:10:38.053 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57531.log 2026-02-22T22:10:38.053 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.65640.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70861.log 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.65640.log.gz 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57531.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85162.log 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57531.log.gz 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.70861.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.90162.log 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88883.log 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85162.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.70861.log.gz 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.90162.log: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.90162.log.gzgzip 2026-02-22T22:10:38.054 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78751.log 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88883.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72538.log 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.85162.log.gz 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78751.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88152.log 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.88883.log.gz 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.78751.log.gz 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72538.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67491.log 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.72538.log.gz 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88152.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.86820.log 2026-02-22T22:10:38.055 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88152.log.gz 2026-02-22T22:10:38.056 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67491.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80889.log 2026-02-22T22:10:38.056 INFO:teuthology.orchestra.run.trial104.stderr: 88.6% -- replaced with /var/log/ceph/cluster1-client.admin.67491.log.gz 2026-02-22T22:10:38.056 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.86820.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94992.log 2026-02-22T22:10:38.056 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80889.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.73442.log 2026-02-22T22:10:38.056 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80889.log.gz 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94992.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.1.54519.log 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.86820.log.gz 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.94992.log.gz 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.73442.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80868.log 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.73442.log.gz 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.1.54519.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57935.log 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80868.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91412.log 2026-02-22T22:10:38.057 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80868.log.gz 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.57935.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48867.log 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.57935.log.gz 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91412.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.56340.log 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.91412.log.gz 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.48867.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48867.log.gz 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.90171.log 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.56340.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.64980.log 2026-02-22T22:10:38.058 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.56340.log.gz 2026-02-22T22:10:38.059 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.0.90171.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52849.log 2026-02-22T22:10:38.059 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.64980.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.70132.log 2026-02-22T22:10:38.059 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.64980.log.gz 2026-02-22T22:10:38.059 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52849.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88090.log 2026-02-22T22:10:38.059 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.52849.log.gz 2026-02-22T22:10:38.059 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.70132.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91594.log 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88090.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.70132.log.gz 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.70740.log 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr: 84.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88090.log.gz 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91594.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.63676.log 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.91594.log.gz 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.70740.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.58171.log 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.0.90171.log.gz 2026-02-22T22:10:38.060 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.70740.log.gz 2026-02-22T22:10:38.061 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.63676.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80709.log 2026-02-22T22:10:38.061 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.58171.log: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.63676.log.gz 2026-02-22T22:10:38.061 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.83613.log 2026-02-22T22:10:38.061 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.58171.log.gz 2026-02-22T22:10:38.061 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80709.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.63814.log 2026-02-22T22:10:38.061 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.83613.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77948.log 2026-02-22T22:10:38.061 INFO:teuthology.orchestra.run.trial104.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.80709.log.gz 2026-02-22T22:10:38.062 INFO:teuthology.orchestra.run.trial104.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.83613.log.gz 2026-02-22T22:10:38.062 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.63814.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60543.log 2026-02-22T22:10:38.062 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.63814.log.gz 2026-02-22T22:10:38.062 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77948.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.93463.log 2026-02-22T22:10:38.062 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60543.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57496.log 2026-02-22T22:10:38.062 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.60543.log.gz 2026-02-22T22:10:38.062 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.93463.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.88704.log 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr: 84.6% -- replaced with /var/log/ceph/cluster2-client.mirror.93463.log.gz 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr: 90.7% -- replaced with /var/log/ceph/cluster2-client.mirror.77948.log.gz 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57496.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74408.log 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.57496.log.gz 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.88704.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.88759.log 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.74408.log: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.88704.log.gz 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.3.49493.log 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.74408.log.gz 2026-02-22T22:10:38.063 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.88759.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.72631.log 2026-02-22T22:10:38.064 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.3.49493.log: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.3.49493.log.gz 2026-02-22T22:10:38.064 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.61088.log 2026-02-22T22:10:38.064 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.72631.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65403.log 2026-02-22T22:10:38.064 INFO:teuthology.orchestra.run.trial104.stderr: 87.0% -- replaced with /var/log/ceph/cluster1-client.admin.72631.log.gz 2026-02-22T22:10:38.064 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.61088.log: 92.2% -- replaced with /var/log/ceph/cluster1-client.mirror.88759.log.gz 2026-02-22T22:10:38.064 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.94605.log 2026-02-22T22:10:38.064 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.61088.log.gz 2026-02-22T22:10:38.065 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65403.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.57217.log 2026-02-22T22:10:38.065 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.65403.log.gz 2026-02-22T22:10:38.065 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.94605.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.94605.log.gz 2026-02-22T22:10:38.065 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58304.log 2026-02-22T22:10:38.065 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.57217.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.59183.log 2026-02-22T22:10:38.065 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.57217.log.gz 2026-02-22T22:10:38.065 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58304.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85118.log 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.58304.log.gz 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.59183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.70498.log 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.59183.log.gz 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85118.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.53381.log 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.70498.log: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.85118.log.gz 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.94109.log 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.70498.log.gz 2026-02-22T22:10:38.066 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.53381.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.91778.log 2026-02-22T22:10:38.067 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.0.94109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80797.log 2026-02-22T22:10:38.067 INFO:teuthology.orchestra.run.trial104.stderr: 89.0% -- replaced with /var/log/ceph/cluster2-client.mirror.53381.log.gz 2026-02-22T22:10:38.067 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.91778.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.55112.log 2026-02-22T22:10:38.067 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80797.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.85439.log 2026-02-22T22:10:38.067 INFO:teuthology.orchestra.run.trial104.stderr: 86.6% -- replaced with /var/log/ceph/cluster1-client.mirror.80797.log.gz 2026-02-22T22:10:38.067 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.55112.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67041.log 2026-02-22T22:10:38.068 INFO:teuthology.orchestra.run.trial104.stderr: 92.9% -- replaced with /var/log/ceph/cluster1-client.mirror.91778.log.gz 2026-02-22T22:10:38.068 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.85439.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.90847.log 2026-02-22T22:10:38.068 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67041.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87816.log 2026-02-22T22:10:38.068 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.67041.log.gz 2026-02-22T22:10:38.068 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.90847.log: 89.7% -- replaced with /var/log/ceph/cluster2-client.mirror.85439.log.gz 2026-02-22T22:10:38.068 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84898.log 2026-02-22T22:10:38.068 INFO:teuthology.orchestra.run.trial104.stderr: 88.3% -- replaced with /var/log/ceph/cluster2-client.mirror.90847.log.gz 2026-02-22T22:10:38.069 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87816.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.74561.log 2026-02-22T22:10:38.069 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.87816.log.gz 2026-02-22T22:10:38.069 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84898.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.72341.log 2026-02-22T22:10:38.069 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.74561.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51719.log 2026-02-22T22:10:38.069 INFO:teuthology.orchestra.run.trial104.stderr: 89.2% -- replaced with /var/log/ceph/cluster2-client.mirror.84898.log.gz 2026-02-22T22:10:38.069 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.72341.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.mirror.74561.log.gz 2026-02-22T22:10:38.069 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56375.log 2026-02-22T22:10:38.070 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.72341.log.gz 2026-02-22T22:10:38.070 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.51719.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58483.log 2026-02-22T22:10:38.070 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.56375.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.51719.log.gz 2026-02-22T22:10:38.070 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.85839.log 2026-02-22T22:10:38.070 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58483.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50972.log 2026-02-22T22:10:38.070 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.85839.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.58483.log.gz 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56417.log 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster2-client.mirror.56375.log.gz 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.85839.log.gz 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.50972.log: 94.7% -- replaced with /var/log/ceph/cluster2-client.mirror.1.54519.log.gz 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.55112.log.gz 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.61691.log 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50972.log.gz 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.mirror.0.94109.log.gz 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.56417.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.96349.log 2026-02-22T22:10:38.071 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.61691.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67971.log 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.mirror.56417.log.gz 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.61691.log.gz 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.96349.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.74243.log 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67971.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.56109.log 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr: 89.1% -- replaced with /var/log/ceph/cluster1-client.mirror.96349.log.gz 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster2-client.admin.67971.log.gz 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.74243.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.2.54520.log 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.74243.log.gz 2026-02-22T22:10:38.072 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.56109.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.67668.log 2026-02-22T22:10:38.073 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.2.54520.log: 89.1% -- replaced with /var/log/ceph/cluster2-client.mirror.56109.log.gz 2026-02-22T22:10:38.073 INFO:teuthology.orchestra.run.trial104.stderr: 19.0% -- replaced with /var/log/ceph/cluster2-client.mirror.2.54520.log.gz 2026-02-22T22:10:38.073 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80073.log 2026-02-22T22:10:38.073 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.67668.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.73567.log 2026-02-22T22:10:38.073 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80073.log: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.67668.log.gz 2026-02-22T22:10:38.073 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.56473.log 2026-02-22T22:10:38.073 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.73567.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.56698.log 2026-02-22T22:10:38.074 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.73567.log.gz 2026-02-22T22:10:38.074 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.56473.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.67904.log 2026-02-22T22:10:38.074 INFO:teuthology.orchestra.run.trial104.stderr: 92.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80073.log.gz 2026-02-22T22:10:38.074 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.56473.log.gz 2026-02-22T22:10:38.074 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.56698.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.66344.log 2026-02-22T22:10:38.074 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.67904.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.69865.log 2026-02-22T22:10:38.074 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.67904.log.gz 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.66344.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.62865.log 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.66344.log.gz 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.69865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78924.log 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.69865.log.gz 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.62865.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.75613.log 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.62865.log.gz 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.92782.log 2026-02-22T22:10:38.075 INFO:teuthology.orchestra.run.trial104.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78924.log.gz 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.75613.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88392.log 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.75613.log.gz 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.92782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.86073.log 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.92782.log.gz 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88392.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43746.log 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.88392.log.gz 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.86073.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.77980.log 2026-02-22T22:10:38.076 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.43746.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43746.log.gz 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87291.log 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.77980.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79070.log 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87291.log: 53.5% -- replaced with /var/log/ceph/cluster2-client.mirror.87291.log.gz 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.80913.log 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr: 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.77980.log.gz 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79070.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93596.log 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr: 84.5% 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.79070.log.gz -- replaced with /var/log/ceph/cluster1-client.mirror.56698.log.gz 2026-02-22T22:10:38.077 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:38.078 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.80913.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.60658.log 2026-02-22T22:10:38.078 INFO:teuthology.orchestra.run.trial104.stderr: 84.5% -- replaced with /var/log/ceph/cluster2-client.mirror.80913.log.gz 2026-02-22T22:10:38.078 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.93596.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.67728.log 2026-02-22T22:10:38.078 INFO:teuthology.orchestra.run.trial104.stderr: 88.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93596.log.gz 2026-02-22T22:10:38.078 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.60658.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.80600.log 2026-02-22T22:10:38.078 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.60658.log.gz 2026-02-22T22:10:38.078 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.67728.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.93924.log 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.80600.log: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.67728.log.gz 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.2.86176.log 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.mirror.80600.log.gz 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.93924.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.65672.log 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.93924.log.gz 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.2.86176.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.2.86176.log.gz 2026-02-22T22:10:38.079 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.65672.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.77217.log 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr: 88.5% -- replaced with /var/log/ceph/cluster1-client.admin.65672.log.gz 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.94213.log 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.77217.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.79038.log 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.94213.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.87974.log 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr: 90.2% -- replaced with /var/log/ceph/cluster1-client.mirror.94213.log.gz 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr: 93.8% -- replaced with /var/log/ceph/cluster2-client.mirror.86073.log.gz 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.77217.log.gz 2026-02-22T22:10:38.080 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.79038.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49896.log 2026-02-22T22:10:38.081 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.87974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.88469.log 2026-02-22T22:10:38.081 INFO:teuthology.orchestra.run.trial104.stderr: 86.8% -- replaced with /var/log/ceph/cluster1-client.mirror.87974.log.gz 2026-02-22T22:10:38.081 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.49896.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.84964.log 2026-02-22T22:10:38.081 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.88469.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.89632.log 2026-02-22T22:10:38.081 INFO:teuthology.orchestra.run.trial104.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.88469.log.gz 2026-02-22T22:10:38.081 INFO:teuthology.orchestra.run.trial104.stderr: 90.6% -- replaced with /var/log/ceph/cluster2-client.mirror.79038.log.gz 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.84964.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.58029.log 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.89632.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.84964.log.gz 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49341.log 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.89632.log.gz 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.87308.log 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.58029.log: /var/log/ceph/cluster2-client.mirror.49341.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.57084.log 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.58029.log.gz 2026-02-22T22:10:38.082 INFO:teuthology.orchestra.run.trial104.stderr: 56.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49341.log.gz 2026-02-22T22:10:38.083 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.87308.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.68037.log 2026-02-22T22:10:38.083 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.57084.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.70669.log 2026-02-22T22:10:38.083 INFO:teuthology.orchestra.run.trial104.stderr: 89.6% -- replaced with /var/log/ceph/cluster2-client.mirror.87308.log.gz 2026-02-22T22:10:38.083 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.68037.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.83479.log 2026-02-22T22:10:38.083 INFO:teuthology.orchestra.run.trial104.stderr: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.68037.log.gz 2026-02-22T22:10:38.083 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.70669.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.93704.log 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.70669.log.gz 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr: 91.6% -- replaced with /var/log/ceph/cluster2-client.mirror.57084.log.gz 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.83479.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.64539.log 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.93704.log: 88.1% -- replaced with /var/log/ceph/cluster1-client.admin.83479.log.gz 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52207.log 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.admin.64539.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78226.log 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.49896.log.gz 2026-02-22T22:10:38.084 INFO:teuthology.orchestra.run.trial104.stderr: 88.2% -- replaced with /var/log/ceph/cluster2-client.admin.64539.log.gz 2026-02-22T22:10:38.085 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.admin.52207.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.89172.log 2026-02-22T22:10:38.085 INFO:teuthology.orchestra.run.trial104.stderr: 91.9% -- replaced with /var/log/ceph/cluster1-client.mirror.93704.log.gz 2026-02-22T22:10:38.085 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78226.log: 90.1%gzip -- replaced with /var/log/ceph/cluster1-client.admin.52207.log.gz 2026-02-22T22:10:38.085 INFO:teuthology.orchestra.run.trial104.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.78946.log 2026-02-22T22:10:38.085 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster2-client.mirror.89172.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.54039.log 2026-02-22T22:10:38.086 INFO:teuthology.orchestra.run.trial104.stderr:/var/log/ceph/cluster1-client.mirror.78946.log: /var/log/ceph/cluster2-client.mirror.54039.log: 90.7% -- replaced with /var/log/ceph/cluster1-client.mirror.78946.log.gz 2026-02-22T22:10:38.086 INFO:teuthology.orchestra.run.trial104.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.mirror.54039.log.gz 2026-02-22T22:10:38.086 INFO:teuthology.orchestra.run.trial104.stderr: 90.6% -- replaced with /var/log/ceph/cluster1-client.mirror.78226.log.gz 2026-02-22T22:10:38.088 INFO:teuthology.orchestra.run.trial104.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.89172.log.gz 2026-02-22T22:10:38.094 INFO:teuthology.orchestra.run.trial104.stderr: 92.4% -- replaced with /var/log/ceph/cluster1-client.mirror.0.50528.log.gz 2026-02-22T22:10:38.098 INFO:teuthology.orchestra.run.trial104.stderr: 94.9% -- replaced with /var/log/ceph/cluster2-client.mirror.90199.log.gz 2026-02-22T22:10:38.181 INFO:teuthology.orchestra.run.trial048.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-22T22:10:38.233 INFO:teuthology.orchestra.run.trial104.stderr: 94.9% -- replaced with /var/log/ceph/cluster1-client.mirror.1.90199.log.gz 2026-02-22T22:10:38.381 INFO:teuthology.orchestra.run.trial104.stderr: 91.5% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-22T22:10:41.043 INFO:teuthology.orchestra.run.trial048.stderr: 94.7% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-22T22:10:41.424 INFO:teuthology.orchestra.run.trial048.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-22T22:10:41.623 INFO:teuthology.orchestra.run.trial048.stderr: 94.8% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-22T22:10:41.624 INFO:teuthology.orchestra.run.trial048.stderr: 2026-02-22T22:10:41.624 INFO:teuthology.orchestra.run.trial048.stderr:real 0m3.946s 2026-02-22T22:10:41.624 INFO:teuthology.orchestra.run.trial048.stderr:user 0m11.172s 2026-02-22T22:10:41.624 INFO:teuthology.orchestra.run.trial048.stderr:sys 0m0.552s 2026-02-22T22:10:41.702 INFO:teuthology.orchestra.run.trial104.stderr: 94.2% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-22T22:10:41.760 INFO:teuthology.orchestra.run.trial104.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-22T22:10:41.789 INFO:tasks.daemonwatchdog.daemon_watchdog:daemon cluster1.osd.2 has been restored 2026-02-22T22:10:41.881 INFO:teuthology.orchestra.run.trial104.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-22T22:10:41.884 INFO:teuthology.orchestra.run.trial104.stderr: 2026-02-22T22:10:41.884 INFO:teuthology.orchestra.run.trial104.stderr:real 0m4.181s 2026-02-22T22:10:41.884 INFO:teuthology.orchestra.run.trial104.stderr:user 0m13.132s 2026-02-22T22:10:41.884 INFO:teuthology.orchestra.run.trial104.stderr:sys 0m1.327s 2026-02-22T22:10:41.884 INFO:tasks.ceph:Archiving logs... 2026-02-22T22:10:41.885 DEBUG:teuthology.misc:Transferring archived files from trial048:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/remote/trial048/log 2026-02-22T22:10:41.887 DEBUG:teuthology.orchestra.run.trial048:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:10:42.795 DEBUG:teuthology.misc:Transferring archived files from trial104:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/remote/trial104/log 2026-02-22T22:10:42.795 DEBUG:teuthology.orchestra.run.trial104:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:10:44.494 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-22T22:10:44.506 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-22T22:10:44.506 DEBUG:teuthology.orchestra.run.trial048:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:10:44.509 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:10:44.673 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-22T22:10:44.685 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-22T22:10:44.685 DEBUG:teuthology.orchestra.run.trial048:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:10:44.687 DEBUG:teuthology.orchestra.run.trial104:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:10:44.702 INFO:teuthology.orchestra.run.trial048.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:10:44.703 INFO:teuthology.orchestra.run.trial104.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:10:44.723 INFO:teuthology.orchestra.run.trial104.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:10:44.723 INFO:teuthology.orchestra.run.trial104.stdout:=============================================================================== 2026-02-22T22:10:44.723 INFO:teuthology.orchestra.run.trial104.stdout:^+ ntp6.kernfusion.at 2 7 377 43 +763us[ +710us] +/- 46ms 2026-02-22T22:10:44.723 INFO:teuthology.orchestra.run.trial104.stdout:^- 170-187-147-56.ip.linode> 2 6 377 43 -5588us[-5641us] +/- 79ms 2026-02-22T22:10:44.723 INFO:teuthology.orchestra.run.trial104.stdout:^* 38.45.64.130 2 8 377 42 -71us[ -124us] +/- 18ms 2026-02-22T22:10:44.723 INFO:teuthology.orchestra.run.trial104.stdout:^+ as393746.mci.trtnw.net 2 7 377 101 -695us[ -751us] +/- 23ms 2026-02-22T22:10:44.733 INFO:teuthology.orchestra.run.trial048.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:10:44.733 INFO:teuthology.orchestra.run.trial048.stdout:=============================================================================== 2026-02-22T22:10:44.733 INFO:teuthology.orchestra.run.trial048.stdout:^- us-central-1.clearnet.pw 2 7 377 100 +7511us[+7468us] +/- 73ms 2026-02-22T22:10:44.733 INFO:teuthology.orchestra.run.trial048.stdout:^+ 172-238-164-57.ip.linode> 4 7 377 105 -3107us[-3133us] +/- 32ms 2026-02-22T22:10:44.733 INFO:teuthology.orchestra.run.trial048.stdout:^* s2-a.time.mci1.us.rozint> 2 6 377 37 +1234us[+1191us] +/- 24ms 2026-02-22T22:10:44.733 INFO:teuthology.orchestra.run.trial048.stdout:^+ LAX.CALTICK.NET 2 8 377 40 +909us[ +866us] +/- 33ms 2026-02-22T22:10:44.734 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-22T22:10:44.745 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-22T22:10:44.745 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-22T22:10:44.759 DEBUG:teuthology.orchestra.run.trial048:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:10:44.830 DEBUG:teuthology.orchestra.run.trial104:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:10:44.886 DEBUG:teuthology.orchestra.run.trial048:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:10:44.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:10:44.913 DEBUG:teuthology.orchestra.run.trial104:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:10:44.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:10:44.939 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-22T22:10:44.953 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-22T22:10:44.966 INFO:teuthology.task.internal:Duration was 1825.189166 seconds 2026-02-22T22:10:44.967 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-22T22:10:44.986 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-22T22:10:44.986 DEBUG:teuthology.orchestra.run.trial048:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:10:44.989 DEBUG:teuthology.orchestra.run.trial104:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:10:45.024 INFO:teuthology.orchestra.run.trial048.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:10:45.027 INFO:teuthology.orchestra.run.trial104.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:10:45.268 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-22T22:10:45.268 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial048.front.sepia.ceph.com 2026-02-22T22:10:45.268 DEBUG:teuthology.orchestra.run.trial048:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:10:45.327 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial104.front.sepia.ceph.com 2026-02-22T22:10:45.327 DEBUG:teuthology.orchestra.run.trial104:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:10:45.347 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-22T22:10:45.347 DEBUG:teuthology.orchestra.run.trial048:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:10:45.369 DEBUG:teuthology.orchestra.run.trial104:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:10:45.448 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-22T22:10:45.449 DEBUG:teuthology.orchestra.run.trial048:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:10:45.451 DEBUG:teuthology.orchestra.run.trial104:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:10:45.473 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:10:45.473 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:10:45.473 INFO:teuthology.orchestra.run.trial048.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:10:45.473 INFO:teuthology.orchestra.run.trial048.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:10:45.474 INFO:teuthology.orchestra.run.trial048.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:10:45.475 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:10:45.476 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:10:45.477 INFO:teuthology.orchestra.run.trial104.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:10:45.477 INFO:teuthology.orchestra.run.trial104.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:10:45.477 INFO:teuthology.orchestra.run.trial104.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:10:45.483 INFO:teuthology.orchestra.run.trial048.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.2% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:10:45.486 INFO:teuthology.orchestra.run.trial104.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.5% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:10:45.488 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-22T22:10:45.505 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-22T22:10:45.505 DEBUG:teuthology.orchestra.run.trial048:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:10:45.551 DEBUG:teuthology.orchestra.run.trial104:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:10:45.575 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-22T22:10:45.594 DEBUG:teuthology.orchestra.run.trial048:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:10:45.597 DEBUG:teuthology.orchestra.run.trial104:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:10:45.619 INFO:teuthology.orchestra.run.trial048.stdout:kernel.core_pattern = core 2026-02-22T22:10:45.639 INFO:teuthology.orchestra.run.trial104.stdout:kernel.core_pattern = core 2026-02-22T22:10:45.654 DEBUG:teuthology.orchestra.run.trial048:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:10:45.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:10:45.694 DEBUG:teuthology.orchestra.run.trial104:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:10:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:10:45.710 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-22T22:10:45.728 INFO:teuthology.task.internal:Transferring archived files... 2026-02-22T22:10:45.729 DEBUG:teuthology.misc:Transferring archived files from trial048:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/remote/trial048 2026-02-22T22:10:45.729 DEBUG:teuthology.orchestra.run.trial048:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:10:45.773 DEBUG:teuthology.misc:Transferring archived files from trial104:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64096/remote/trial104 2026-02-22T22:10:45.773 DEBUG:teuthology.orchestra.run.trial104:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:10:45.811 INFO:teuthology.task.internal:Removing archive directory... 2026-02-22T22:10:45.811 DEBUG:teuthology.orchestra.run.trial048:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:10:45.814 DEBUG:teuthology.orchestra.run.trial104:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:10:45.861 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-22T22:10:45.870 INFO:teuthology.task.internal:Not uploading archives. 2026-02-22T22:10:45.870 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-22T22:10:45.882 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-22T22:10:45.882 DEBUG:teuthology.orchestra.run.trial048:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:10:45.883 DEBUG:teuthology.orchestra.run.trial104:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:10:45.898 INFO:teuthology.orchestra.run.trial048.stdout: 917572 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:10 /home/ubuntu/cephtest 2026-02-22T22:10:45.918 INFO:teuthology.orchestra.run.trial104.stdout: 917521 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:10 /home/ubuntu/cephtest 2026-02-22T22:10:45.919 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-22T22:10:45.937 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-22T22:10:45.979 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-bitmap supported-random-distro$/{centos_latest} workloads/rbd-mirror-workunit-policy-none} duration: 1825.1891655921936 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-22T22:10:45.980 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:10:46.091 INFO:teuthology.run:pass